Segmentation Fault with Projection API

[davidteofilovic]

Hey guys,

I believe I have set everything up correctly, but the only response I'm getting from the console when I run the script is "Segmentation fault". Here are the versions I'm running:

Software Versioning

mysql-js: up-to-date as of August 5th, 2016
node version: v0.12.2
MySQL Cluster version: 7.5.3

MySql Cluster Table Set-up

CREATE TABLE 'graph'.'objects' (  'id_company' INT NOT NULL COMMENT '',  'id' BIGINT(20) NOT NULL COMMENT '',  'otype' VARCHAR(45) NOT NULL COMMENT '',  'data' JSON, 'data2' JSON,  PRIMARY KEY ('id_company','id', 'otype')  COMMENT '')  ENGINE=NDBCLUSTER  PARTITION BY KEY(id_company);

CREATE TABLE 'graph'.'associations' (  'id_company' INT NOT NULL COMMENT '',  'id1' BIGINT(20) NOT NULL COMMENT '',  'atype' BIGINT(20) NOT NULL COMMENT '',  'id2' BIGINT(20) NOT NULL COMMENT '',  'time' DATETIME NULL DEFAULT NOW() COMMENT '',  'data' JSON ,  PRIMARY KEY ('id_company', 'id1', 'atype', 'id2')  COMMENT '')  ENGINE=NDBCLUSTER  PARTITION BY KEY(id_company);

Objects has a one-to-many relationship with associations.

App.js Code

"use strict";
var jones = require("/api_tests/mysql-js/database-jones/");
var connectionProperties = new jones.ConnectionProperties("ndb");


function Objects(id, id_company, otype, data, data2) {
        this.id = id;
        this.id_company = id_company;
        this.otype = otype;
        this.data = data;
        this.data2 = data2;
}

function Associations(id_company, id1, atype, id2, time, data) {
        this.id_company = id_company;
        this.id1 = id1;
        this.atype = atype;
        this.id2 = id2;
        this.time = time;
        this.data = data;
}

function mapObjects() {
    var objectMapping = new jones.TableMapping("objects");
        objectMapping.mapField("id");
        objectMapping.mapField("id_company");
        objectMapping.mapField("otype");
        objectMapping.mapField("data");
        objectMapping.mapField("data2");

        objectMapping.mapOneToMany({
            fieldName: "assoc",
            target: Associations,
            targetField: "obj_assoc"
        });

    objectMapping.applyToClass(Objects);
}

function mapAssociations() {
    var associationMapping = new jones.TableMapping("associations");
        associationMapping.mapField("id_company");
        associationMapping.mapField("id1");
        associationMapping.mapField("atype");
        associationMapping.mapField("id2");
        associationMapping.mapField("time");
        associationMapping.mapField("data");


        associationMapping.mapManyToOne({
            fieldName: "obj_assoc",
            target: Objects,
            foreignKey: "fk_id1"
        });

    associationMapping.applyToClass(Associations);
}

function mapShop() {
    mapObjects();
    mapAssociations();
}

mapShop();

var associationProject = new jones.Projection(Associations)
.addFields("id_company", "id1", "atype", "id2", "time", "data");

var objectProjection = new jones.Projection(Objects)
.addFields("id", "id_company", "otype", "data")
.addRelationship("assoc", associationProject);

var onFind = function(err, result) {
    if (err) { console.log(err);} else {
        console.log(result);
    }
};

jones.openSession(connectionProperties).then(
    function(session) {
        session.find(objectProjection, {id: 1}, onFind);
    }
);

[jdduncan]

I'll take a look this afternoon!

JD

On Aug 10, 2016, at 11:17 AM, davidteofilovic [email protected] wrote:

Hey guys,

I believe I have set everything up correctly, but the only response I'm getting from the console when I run the script is "Segmentation fault". Here are the versions I'm running:

Software Versioning

mysql-js: up-to-date as of August 5th, 2016
node version: v0.12.2
MySQL Cluster version: 7.5.3

MySql Cluster Table Set-up

CREATE TABLE 'graph'.'objects' (
'id_company' INT NOT NULL COMMENT '',
'id' BIGINT(20) NOT NULL COMMENT '',
'otype' VARCHAR(45) NOT NULL COMMENT '',
'data' JSON,
'data2' JSON,
PRIMARY KEY ('id_company','id', 'otype') COMMENT '')
ENGINE=NDBCLUSTER
PARTITION BY KEY(id_company);

CREATE TABLE 'graph'.'associations' (
'id_company' INT NOT NULL COMMENT '',
'id1' BIGINT(20) NOT NULL COMMENT '',
'atype' BIGINT(20) NOT NULL COMMENT '',
'id2' BIGINT(20) NOT NULL COMMENT '',
'time' DATETIME NULL DEFAULT NOW() COMMENT '',
'data' JSON ,
PRIMARY KEY ('id_company', 'id1', 'atype', 'id2') COMMENT '')
ENGINE=NDBCLUSTER
PARTITION BY KEY(id_company);

Objects has a one-to-many relationship with associations.

App.js Code

"use strict"
;

var jones = require("/api_tests/mysql-js/database-jones/"
);

var connectionProperties = new jones.ConnectionProperties("ndb"
);

function Objects(id, id_company, otype, data, data2
) {

this.id =
id;

this.id_company =
id_company;

this.otype =
otype;

this.data =
data;

this.data2 =
data2;
}

function Associations(id_company, id1, atype, id2, time, data
) {

this.id_company =
id_company;

this.id1 =
id1;

this.atype =
atype;

this.id2 =
id2;

this.time =
time;

this.data =
data;
}

function mapObjects
() {

var objectMapping = new jones.TableMapping("objects"
);

objectMapping.mapField("id"
);

objectMapping.mapField("id_company"
);

objectMapping.mapField("otype"
);

objectMapping.mapField("data"
);

objectMapping.mapField("data2"
);

objectMapping.mapOneToMany
({
fieldName
: "assoc"
,
target
:
Associations,
targetField
: "obj_assoc"

    });

objectMapping.applyToClass
(Objects);
}

function mapAssociations
() {

var associationMapping = new jones.TableMapping("associations"
);

associationMapping.mapField("id_company"
);

associationMapping.mapField("id1"
);

associationMapping.mapField("atype"
);

associationMapping.mapField("id2"
);

associationMapping.mapField("time"
);

associationMapping.mapField("data"
);

associationMapping.mapManyToOne
({
fieldName
: "obj_assoc"
,
target
:
Objects,
foreignKey
: "fk_id1"

    });

associationMapping.applyToClass
(Associations);
}

function mapShop
() {

mapObjects
();

mapAssociations
();
}

mapShop
();

var associationProject = new jones.Projection
(Associations)
.
addFields("id_company", "id1", "atype", "id2", "time", "data"
);

var objectProjection = new jones.Projection
(Objects)
.
addFields("id", "id_company", "otype", "data"
)
.
addRelationship("assoc"
, associationProject);

var onFind = function(err, result
) {

if (err) { console.log(err);} else
{

console.log
(result);
}
};

jones.openSession(connectionProperties).then
(

function(session
) {

session.find(objectProjection, {id: 1
}, onFind);
}
);

—
You are receiving this because you are subscribed to this thread.
Reply to this email directly, view it on GitHub, or mute the thread.

[CraigLRussell]

Currently, we restrict relationships to use foreign keys. There is no foreign key called fk_id1. There should be a better error message.

I'll create a test case for this.

How should the foreign key fk_id1 be defined?

[davidteofilovic]

Apologies! I omitted that from my original post. We have a foreign key constraint called "fk_id" that makes "id1" on the associations table a foreign key that references "id" on the objects table.

[jdduncan]

I don't see the segfault, but I do (without the foreign key) see this error message:

{ [Error:
Bad relationship field mapping; foreign key fk_id1 does not exist in table; possible foreign keys are: ] sqlstate: 'HY000' }

I also have a hunch that PARTITION BY KEY may cause a problem, so I'd suggest removing PARTITION BY KEY from both tables, getting the app to work, and then putting it back and testing again. (The issue here is just that the partition key and the primary key in NDB are usually interchangable, but we don't currently have tests for cases where they are not, just because effectively testing it requires a larger cluster than we have in our usual dev environments).

[davidteofilovic]

Right that's the error I get when I put the incorrect foreign key as well. Again sorry for omitting that from the original post. I'm curious to see what happens for you when you make "id1" on the associations table a foreign key that references the "id" column on the objects table. In the mean time I'll try and remove the partition and see if that changes things. Thanks!

[davidteofilovic]

John,

Your hunch was right and the partitions were the root cause of my issue. Removing them and switching the connection properties from ndb to mysql made everything work perfectly! As soon as I re-added the partition i got the following error:

/api_tests/mysql-js/node_modules/mysql/lib/protocol/Parser.js:78 throw err; // Rethrow non-MySQL errors
^ TypeError: Cannot read property 'fieldName' of undefined

[CraigLRussell]

Ok. We will add a test case that includes partition by key.
Does the error occur when you add partition by key to either of the tables? Or just the table with the foreign key?
Thanks for the report.

[jdduncan]

Hi David,

Two questions here!

1. Can you post your most current SHOW CREATE TABLE for the two tables?

2. What happens when you try an ndb connection without any PARTITION clause on the tables?

[davidteofilovic]

Okay so in order:

Craig, I removed the partition on the table with the foreign key (keeping the one on referencing table) and it still gave me issues. After I removed the referencing table's partitioning it worked again.

John here are my current show create tables (without partitioning on id_company):
CREATE TABLEobjects( id_companyint(11) NOT NULL, idbigint(20) NOT NULL AUTO_INCREMENT, otypevarchar(45) NOT NULL, datajson DEFAULT NULL, data2varchar(2000) DEFAULT NULL, PRIMARY KEY (id,id_company), UNIQUE KEYid(id) ) ENGINE=ndbcluster AUTO_INCREMENT=14066 DEFAULT CHARSET=utf8

CREATE TABLEassociations( id_companyint(11) NOT NULL, id1bigint(20) NOT NULL, atypevarchar(45) NOT NULL, id2bigint(20) NOT NULL, timetimestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, datajson DEFAULT NULL, PRIMARY KEY (id_company,id1,atype,id2), KEYidx_associations_id1(id1), CONSTRAINTfk_id1FOREIGN KEY (id1) REFERENCESobjects(id) ON DELETE NO ACTION ON UPDATE NO ACTION ) ENGINE=ndbcluster DEFAULT CHARSET=utf8

2. When I switched it to ndb from the working mysql it gives me the following text:

node: ../impl/src/ndb/NdbRecordObject.cpp:76: NdbRecordObject::NdbRecordObject(const Record_, ColumnHandlerSet_, const Arguments&): Assertion `nblobs == record->getNoOfBlobColumns()' failed.
Aborted

[jdduncan]

Hi David,

I have opened up issue #38 for separate tracking of that assertion on "nblobs", and issue #39 for separate tracking of the PARTITION BY KEY problem.

[davidteofilovic]

Awesome thanks! Should I close this issue then?

[jdduncan]

It looks to me like the example code hangs after fetching the result. You can call session.close() at the end of a session, and jones.closeAllOpenSessionFactories() at the end of the script, to disconnect from the database and allow the session to close.

If everything here is resolved you are welcome to close the issue, and I'll keep working on 38.

[davidteofilovic]

Hi John,

I changed my openSession function block to be

jones.openSession(connectionProperties).then(
    function(session) {
        return session.find(objectProjection, {id: 1});
    }).
    then(console.log, console.trace).
    then(jones.closeAllOpenSessionFactories).
    then(process.exit, console.trace);

and with ndb as the connection property I'm getting issue 38. With 'mysql' I'm getting the

/api_tests/mysql-js/node_modules/mysql/lib/protocol/Parser.js:78
        throw err; // Rethrow non-MySQL errors
              ^
TypeError: Cannot read property 'fieldName' of undefined
    at DBTableHandlerPrivate.addField (/api_tests/mysql-js/d

[jdduncan]

Here's a bit about the specific problem in #38.

When you create a column of type JSON in MySQL, you don't have a chance to specify a maximum length for the column. So the basic type of a JSON column is actually BLOB, and the JSON content is stored using MySQL's wl#8132 binary encoding for JSON.

Unfortunately, support for BLOB columns is limited in some ways, and one of the limits in NDB is that the push-down-join interface (which we call "SPJ", for Select/Project/Join) does not support them. Because jones-ndb uses SPJ to support the join query on Objects and Associations, it cannot both execute the join and fetch the BLOB (or JSON) columns in a single operation.

There are many possible work-arounds. Here are a few that come to mind:

• If you know the maximum size of the JSON document, and know that it will always fit in a VARCHAR column, then use VARCHAR (with the Jones JSON Converter) or VARBINARY (with the Jones wl8132 converter) instead of a JSON column.
• Use mysql rather than ndb. But note that ndb will always be faster, and the mysql server is also subject to the limitations of SPJ and required to work around them internally, as you would be able to see by running EXPLAIN on the joins.
• Run the join operation using a projection that includes only the key columns, and then use session.load() or batch.load() to bring the rest of the objects including the BLOBs into memory. If I were to work around the problem internally in jones-ndb, this is the strategy I would use.

[davidteofilovic]

Hey John,

Thank you for looking into this for me. I tried to do the third work-around by omitting the JSON columns from the addfields() function, but I'm still getting the same error with the assertion failing. Do I need to get the latest version from Github to make that work-around work?

Thanks

[jdduncan]

No, I don't think you need to update. I expect you to be able to run the join if the mappings & projections do not include the JSON columns. But I also see that it's not working as expected, so I'll look into it.

[CraigLRussell]

I'm going to look into the mysql bug issue reported above:

/api_tests/mysql-js/node_modules/mysql/lib/protocol/Parser.js:78
throw err; // Rethrow non-MySQL errors
^
TypeError: Cannot read property 'fieldName' of undefined
at DBTableHandlerPrivate.addField (/api_tests/mysql-js/d

[jdduncan]

There are a some tricky parts to getting the "load the JSON fields later" workaround right. You have to set mapAllColumns=false on the TableMappings to really exclude the JSON columns; you have to have a "do-nothing" constructor like Object() {} because session.find calls your constructor with no arguments and does not allow you to clobber any of the data that was just read; and you should do the Projection find and the batch load within one transaction so that they see a consistent view of the data. Later this morning when I get a chance I will work up a code example for you showing all of this together.

[CraigLRussell]

I think it would be better if the adapter would "load the JSON fields later". For queries, once the values are loaded, the adapter can internally execute a second operation with the JSON fields that exceed the inline size.

[jdduncan]

Yes, I definitely agree with that.

[davidteofilovic]

Sounds good John, I will wait for your example before I proceed. Thank you guys for all the work you've been putting in to help us out!

[jdduncan]

"use strict";
//require("unified_debug").level_debug();
var jones = require("./database-jones");
var connectionProperties = new jones.ConnectionProperties("ndb");
connectionProperties.database = 'graph';

/* A no-op constructor for object and association that have key fields only.  
   session.find() will call these constructors with no arguments.
   The "Basic" constructor has only key fields mapped.
*/
function BasicObject() {}
function BasicAssociation() {}


/* The "Full" constructor has a default mapping with all fields mapped.
   We will call the constructor with a key-only object.
   Then we can use load() to load the non-key fields from the database.
*/
function FullAssociation(basicAssociation) {
  if(basicAssociation) {
    this.id_company = basicAssociation.id_company;
    this.id1 = basicAssociation.id1;
    this.atype = basicAssociation.atype;
    this.id2 = basicAssociation.id2;
  }
}

function FullObject(basicObject) {
  if(basicObject) {
    this.id = basicObject.id;
    this.id_company = basicObject.id_company;
    this.assoc = [];
    if(Array.isArray(basicObject.assoc)) {
      basicObject.assoc.forEach(function(basicAssoc, n) {
        this.assoc[n] = new FullAssociation(basicAssoc);
      }, this);
    }
  }
}


/* Mappings for the "basic" objects have key fields only */
var objectMapping = new jones.TableMapping("objects");
objectMapping.mapAllColumns = false;   // important so there are no BLOBs!
objectMapping.mapField("id");
objectMapping.mapField("id_company");
objectMapping.mapOneToMany({
  fieldName: "assoc",
  target: BasicAssociation,
  targetField: "obj_assoc"
});
objectMapping.applyToClass(BasicObject);

var associationMapping = new jones.TableMapping("associations");
associationMapping.mapAllColumns = false;
associationMapping.mapField("id_company");
associationMapping.mapField("id1");
associationMapping.mapField("atype");
associationMapping.mapField("id2");

associationMapping.mapManyToOne({
  fieldName: "obj_assoc",
  target: BasicObject,
  foreignKey: "fk_id1"
});
associationMapping.applyToClass(BasicAssociation);


/* Projections: */
var associationProject = new jones.Projection(BasicAssociation)
.addFields("id_company", "id1", "atype", "id2");

var objectProjection = new jones.Projection(BasicObject)
.addFields("id", "id_company").addRelationship("assoc", associationProject);


/* Apply a default mapping to the "full" classes */
new jones.TableMapping("objects").applyToClass(FullObject);
new jones.TableMapping("associations").applyToClass(FullAssociation);

/* Here we go: */
var session, batch, fullObject;

jones.openSession(connectionProperties).then(function(_session) {
    session = _session;
    /* Start a transaction so that the projection read and the batch read
      have a single consistent view of the database  */
    session.currentTransaction().begin();
    return session.find(objectProjection, {id: 1});
}).then(function(result) {
    console.log("Intermediate result:", result);
    /* Build a FullObject from the result, and, in a batch, load the object
       and all its associations 
    */
    fullObject = new FullObject(result);
    batch = session.createBatch();
    batch.load(fullObject);   // load the parent
    fullObject.assoc.forEach(function(assoc) {
      batch.load(assoc);
    });
    return batch.execute();
}).then(function() {
    console.log("Full result:");
    console.log(fullObject);  /* success */
  }, function(error) {
    console.trace(error);     /* failure of any step up to this point */
}).then(function() {
    /* Close the transaction */
    if(session) {
      console.log("Closing transaction.");
      return session.currentTransaction().commit();
    }
}).then(function() {
    jones.closeAllOpenSessionFactories();
});

[davidteofilovic]

That's great thank you so much John! I will use this example and hopefully build off this to meet our needs. Should I close this issue or are you guys still looking into some related bugs?

[CraigLRussell]

partition.txt

This patch will fix the bug:
TypeError: Cannot read property 'fieldName' of undefined

It would fail on using any table defined with PARTITION BY KEY.

[CraigLRussell]

Looks like all issues related to this are resolved.