From 9fc39148f3ebc919d3b5ab4589d3d9a368382f18 Mon Sep 17 00:00:00 2001 From: Jan Dvorak Date: Thu, 6 Oct 2022 10:18:29 +0200 Subject: [PATCH 01/14] Bump minimum Meteor version --- CHANGELOG.md | 3 +++ package.js | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66a4a69e..f8572c10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 1.5.0 +- Minimum Meteor version bumped to 1.12.1 + ## 1.4.0 - Add tests for Meteor 2.6 & 2.7.3 - Migrate TravisCI test to GitHub Actions diff --git a/package.js b/package.js index d5e89262..b9012730 100755 --- a/package.js +++ b/package.js @@ -1,6 +1,6 @@ Package.describe({ name: "cultofcoders:grapher", - version: "1.4.0", + version: "1.5.0", // Brief, one-line summary of the package. summary: "Grapher is a data fetching layer on top of Meteor", // URL to the Git repository containing the source code for this package. @@ -20,7 +20,7 @@ const npmPackages = { Package.onUse(function (api) { Npm.depends(npmPackages); - api.versionsFrom(["1.3", "2.3", "2.6"]); + api.versionsFrom(["1.12.1", "2.3", "2.6", "2.7"]); var packages = [ "ecmascript", From 59c42baed332fcb2e1ecece8d0f81fef106b0f3e Mon Sep 17 00:00:00 2001 From: Jan Dvorak Date: Sat, 31 Dec 2022 16:36:36 +0900 Subject: [PATCH 02/14] Bump minimum version of Meteor to 2.3 for dependencies update --- .github/workflows/test.yml | 2 +- CHANGELOG.md | 3 ++- package.js | 14 +++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bca3fbbf..5ab9a54f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - meteor: [1.12.2, 2.6.1, 2.7.3] + meteor: [2.3, 2.6.1, 2.7.3, 2.9.1] steps: - uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index f8572c10..7d518549 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## 1.5.0 -- Minimum Meteor version bumped to 1.12.1 +- Minimum Meteor version bumped to 2.3 +- Update dependencies ## 1.4.0 - Add tests for Meteor 2.6 & 2.7.3 diff --git a/package.js b/package.js index b9012730..bc2c3d7b 100755 --- a/package.js +++ b/package.js @@ -20,7 +20,7 @@ const npmPackages = { Package.onUse(function (api) { Npm.depends(npmPackages); - api.versionsFrom(["1.12.1", "2.3", "2.6", "2.7"]); + api.versionsFrom(["2.3", "2.6.1", "2.7.3", "2.9.1"]); var packages = [ "ecmascript", @@ -29,11 +29,11 @@ Package.onUse(function (api) { "check", "reactive-var", "mongo", - "matb33:collection-hooks@1.1.2", + "matb33:collection-hooks@1.2.0", "reywood:publish-composite@1.7.3", - "dburles:mongo-collection-instances@0.3.5", + "dburles:mongo-collection-instances@0.3.6", "peerlibrary:subscription-scope@0.5.0", - "herteby:denormalize@0.6.6" + "herteby:denormalize@0.6.7" ]; api.use(packages); @@ -54,10 +54,10 @@ Package.onTest(function (api) { "random", "ecmascript", "underscore", - "matb33:collection-hooks@1.1.0", + "matb33:collection-hooks@1.2.0", "reywood:publish-composite@1.7.3", - "dburles:mongo-collection-instances@0.3.5", - "herteby:denormalize@0.6.6", + "dburles:mongo-collection-instances@0.3.6", + "herteby:denormalize@0.6.7", "mongo" ]; From 04efd154ce49cf1fdcb5f3f44387b2aa3526ed8e Mon Sep 17 00:00:00 2001 From: Jan Dvorak Date: Sun, 7 May 2023 10:44:19 +0200 Subject: [PATCH 03/14] Update test and changelog --- .github/workflows/test.yml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b28371e2..05be803a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - meteor: [2.3, 2.6.1, 2.7.3, 2.9.1] + meteor: [2.3, 2.6.1, 2.7.3, 2.8.1, 2.9.1, 2.12] steps: - uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 34984140..535ddda0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 1.5.0 - Minimum Meteor version bumped to 2.3 - Update dependencies +- Allow unblocking recursive publications [@Floriferous](https://github.com/Floriferous) ## 1.4.1 - Fix reactive counters when filtering on dates [@vparpoil](https://github.com/vparpoil) [PR](https://github.com/cult-of-coders/grapher/pull/402) From e0d20d898e62248eef11896d01ee685ce9ae299f Mon Sep 17 00:00:00 2001 From: Jan Dvorak Date: Sun, 7 May 2023 10:59:55 +0200 Subject: [PATCH 04/14] Change testing targets for 1.5 release --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e31c889..05be803a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - meteor: [1.12.2, 2.6.1, 2.7.3, 2.9.1] + meteor: [2.3, 2.6.1, 2.7.3, 2.8.1, 2.9.1, 2.12] steps: - uses: actions/checkout@v3 From e9a79920e137d299234188c48e880d0451d6a769 Mon Sep 17 00:00:00 2001 From: Jan Dvorak Date: Sun, 7 May 2023 11:04:18 +0200 Subject: [PATCH 05/14] More specific Meteor 2.3 test target --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05be803a..39fb49f5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - meteor: [2.3, 2.6.1, 2.7.3, 2.8.1, 2.9.1, 2.12] + meteor: [2.3.6, 2.6.1, 2.7.3, 2.8.1, 2.9.1, 2.12] steps: - uses: actions/checkout@v3 From 5919308cc4032b93ed5c5afc021eebc538f69a6e Mon Sep 17 00:00:00 2001 From: Jan Dvorak Date: Sun, 7 May 2023 11:04:18 +0200 Subject: [PATCH 06/14] More specific Meteor 2.3 test target --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05be803a..39fb49f5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - meteor: [2.3, 2.6.1, 2.7.3, 2.8.1, 2.9.1, 2.12] + meteor: [2.3.6, 2.6.1, 2.7.3, 2.8.1, 2.9.1, 2.12] steps: - uses: actions/checkout@v3 From 0b91730954484c65dc6693a0803e3a125b91e8ac Mon Sep 17 00:00:00 2001 From: Jan Dvorak Date: Sun, 7 May 2023 11:12:14 +0200 Subject: [PATCH 07/14] Attempting to fix tests --- README.md | 2 +- package.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2c7bd5b6..906c2ac4 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ - Read more about our approach coming from Meteor: https://www.bluelibs.com/blog/2021/11/26/the-meteor-of-2022 - We've implemented [Grapher aka Nova](https://www.bluelibs.com/products/nova) as a standalone npm package compatible to native MongoDB drivers (including Meteor), it is not as feature-rich (no meta links, no pubsub functionality) but is more advanced. -# Grapher 1.4.1 +# Grapher 1.5 _Grapher_ is a Data Fetching Layer on top of Meteor and MongoDB. It is production ready and battle tested. Brought to you by [Cult of Coders](https://www.cultofcoders.com) — Web & Mobile Development Company. diff --git a/package.js b/package.js index bc2c3d7b..c68b583b 100755 --- a/package.js +++ b/package.js @@ -20,7 +20,7 @@ const npmPackages = { Package.onUse(function (api) { Npm.depends(npmPackages); - api.versionsFrom(["2.3", "2.6.1", "2.7.3", "2.9.1"]); + api.versionsFrom(["2.3.6", "2.6.1", "2.7.3", "2.8.1", "2.9.1"]); var packages = [ "ecmascript", From b4d00872c2be84bbec2065e29aa3b7111746f502 Mon Sep 17 00:00:00 2001 From: Jan Dvorak Date: Sun, 7 May 2023 11:25:42 +0200 Subject: [PATCH 08/14] Revert specific 2.3 test --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39fb49f5..05be803a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - meteor: [2.3.6, 2.6.1, 2.7.3, 2.8.1, 2.9.1, 2.12] + meteor: [2.3, 2.6.1, 2.7.3, 2.8.1, 2.9.1, 2.12] steps: - uses: actions/checkout@v3 From b4b71173dc0e6f684f19fd18fef61f5bc2b53d7e Mon Sep 17 00:00:00 2001 From: Jan Dvorak Date: Sun, 7 May 2023 11:27:48 +0200 Subject: [PATCH 09/14] Meteor 2.3.1 --- .github/workflows/test.yml | 2 +- package.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05be803a..fdbd9cb6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - meteor: [2.3, 2.6.1, 2.7.3, 2.8.1, 2.9.1, 2.12] + meteor: [2.3.1, 2.6.1, 2.7.3, 2.8.1, 2.9.1, 2.12] steps: - uses: actions/checkout@v3 diff --git a/package.js b/package.js index c68b583b..5bd77252 100755 --- a/package.js +++ b/package.js @@ -20,7 +20,7 @@ const npmPackages = { Package.onUse(function (api) { Npm.depends(npmPackages); - api.versionsFrom(["2.3.6", "2.6.1", "2.7.3", "2.8.1", "2.9.1"]); + api.versionsFrom(["2.3.1", "2.6.1", "2.7.3", "2.8.1", "2.9.1"]); var packages = [ "ecmascript", From 16127be88b11390c9d9319d4f09f993a9be1db63 Mon Sep 17 00:00:00 2001 From: Berislav Date: Tue, 24 Oct 2023 09:36:35 +0200 Subject: [PATCH 10/14] Reverse link nested objects improvements --- lib/namedQuery/testing/client.test.js | 1 - .../hypernova/assembleAggregateResults.js | 88 ------------- lib/query/hypernova/buildAggregatePipeline.js | 83 ------------ lib/query/hypernova/buildVirtualNodeProps.js | 40 ++++++ lib/query/hypernova/processVirtualNode.js | 99 ++++++++++++++ lib/query/hypernova/storeHypernovaResults.js | 29 ++-- lib/query/testing/server.test.js | 124 ++++++++++++++++++ 7 files changed, 276 insertions(+), 188 deletions(-) delete mode 100644 lib/query/hypernova/assembleAggregateResults.js delete mode 100755 lib/query/hypernova/buildAggregatePipeline.js create mode 100644 lib/query/hypernova/buildVirtualNodeProps.js create mode 100644 lib/query/hypernova/processVirtualNode.js diff --git a/lib/namedQuery/testing/client.test.js b/lib/namedQuery/testing/client.test.js index 3778c3ef..d40b266f 100755 --- a/lib/namedQuery/testing/client.test.js +++ b/lib/namedQuery/testing/client.test.js @@ -204,7 +204,6 @@ describe('Named Query', function () { // since in 2.5+ an actual Map is used const userDocs = Users._collection._docs._map; const docMap = userDocs instanceof Map ? Object.fromEntries(userDocs) : userDocs; - // users collection on the client should have 4 items (user 3 and friends - user 0,1,2) assert.equal(_.keys(docMap).length, 4); diff --git a/lib/query/hypernova/assembleAggregateResults.js b/lib/query/hypernova/assembleAggregateResults.js deleted file mode 100644 index 9e1e5d1a..00000000 --- a/lib/query/hypernova/assembleAggregateResults.js +++ /dev/null @@ -1,88 +0,0 @@ -import sift from 'sift'; -import cleanObjectForMetaFilters from './lib/cleanObjectForMetaFilters'; - -/** - * This only applies to inversed links. It will assemble the data in a correct manner. - */ -export default function(childCollectionNode, aggregateResults, metaFilters) { - const linker = childCollectionNode.linker; - const linkStorageField = linker.linkStorageField; - const linkName = childCollectionNode.linkName; - const isMeta = linker.isMeta(); - const isMany = linker.isMany(); - - let allResults = []; - - if (isMeta && metaFilters) { - const metaFiltersTest = sift(metaFilters); - _.each(childCollectionNode.parent.results, parentResult => { - cleanObjectForMetaFilters( - parentResult, - linkStorageField, - metaFiltersTest - ); - }); - } - - if (isMeta && isMany) { - // This case is treated differently because we get an array response from the pipeline. - - _.each(childCollectionNode.parent.results, parentResult => { - parentResult[linkName] = parentResult[linkName] || []; - - const eligibleAggregateResults = _.filter( - aggregateResults, - aggregateResult => { - return _.contains(aggregateResult._id, parentResult._id); - } - ); - - if (eligibleAggregateResults.length) { - const datas = _.pluck(eligibleAggregateResults, 'data'); /// [ [x1, x2], [x2, x3] ] - - _.each(datas, data => { - _.each(data, item => { - parentResult[linkName].push(item); - }); - }); - } - }); - - _.each(aggregateResults, aggregateResult => { - _.each(aggregateResult.data, item => allResults.push(item)); - }); - } else { - let comparator; - if (isMany) { - comparator = (aggregateResult, result) => - _.contains(aggregateResult._id, result._id); - } else { - comparator = (aggregateResult, result) => - aggregateResult._id == result._id; - } - - const childLinkName = childCollectionNode.linkName; - const parentResults = childCollectionNode.parent.results; - - parentResults.forEach(parentResult => { - // We are now finding the data from the pipeline that is related to the _id of the parent - const eligibleAggregateResults = aggregateResults.filter( - aggregateResult => comparator(aggregateResult, parentResult) - ); - - eligibleAggregateResults.forEach(aggregateResult => { - if (Array.isArray(parentResult[childLinkName])) { - parentResult[childLinkName].push(...aggregateResult.data); - } else { - parentResult[childLinkName] = [...aggregateResult.data]; - } - }); - }); - - aggregateResults.forEach(aggregateResult => { - allResults.push(...aggregateResult.data); - }); - } - - childCollectionNode.results = allResults; -} diff --git a/lib/query/hypernova/buildAggregatePipeline.js b/lib/query/hypernova/buildAggregatePipeline.js deleted file mode 100755 index d5607d82..00000000 --- a/lib/query/hypernova/buildAggregatePipeline.js +++ /dev/null @@ -1,83 +0,0 @@ -import { _ } from 'meteor/underscore'; -import {SAFE_DOTTED_FIELD_REPLACEMENT} from './constants'; - -export default function (childCollectionNode, filters, options, userId) { - let containsDottedFields = false; - const linker = childCollectionNode.linker; - const linkStorageField = linker.linkStorageField; - const collection = childCollectionNode.collection; - - let pipeline = []; - - if (collection.firewall) { - collection.firewall(filters, options, userId); - } - - filters = cleanUndefinedLeafs(filters); - - pipeline.push({$match: filters}); - - if (options.sort && _.keys(options.sort).length > 0) { - pipeline.push({$sort: options.sort}) - } - - let _id = linkStorageField; - if (linker.isMeta()) { - _id += '._id'; - } - - let dataPush = { - _id: '$_id' - }; - - _.each(options.fields, (value, field) => { - if (field.indexOf('.') >= 0) { - containsDottedFields = true; - } - const safeField = field.replace(/\./g, SAFE_DOTTED_FIELD_REPLACEMENT); - dataPush[safeField] = '$' + field - }); - - if (linker.isMeta()) { - dataPush[linkStorageField] = '$' + linkStorageField; - } - - pipeline.push({ - $group: { - _id: "$" + _id, - data: { - $push: dataPush - } - } - }); - - if (options.limit || options.skip) { - let $slice = ["$data"]; - if (options.skip) $slice.push(options.skip); - if (options.limit) $slice.push(options.limit); - - pipeline.push({ - $project: { - _id: 1, - data: {$slice} - } - }) - } - - function cleanUndefinedLeafs(tree) { - const a = Object.assign({}, tree); - _.each(a, (value, key) => { - if (value === undefined) { - delete a[key]; - } - - if (!Array.isArray(value) && _.isObject(value) && !(value instanceof Date)) { - a[key] = cleanUndefinedLeafs(value); - } - }) - - return a; - } - - return {pipeline, containsDottedFields}; -} diff --git a/lib/query/hypernova/buildVirtualNodeProps.js b/lib/query/hypernova/buildVirtualNodeProps.js new file mode 100644 index 00000000..c3b5847a --- /dev/null +++ b/lib/query/hypernova/buildVirtualNodeProps.js @@ -0,0 +1,40 @@ +import { _ } from 'meteor/underscore'; +import { isFieldInProjection } from '../lib/fieldInProjection'; + +export default function (childCollectionNode, filters, options, userId) { + const linker = childCollectionNode.linker; + const linkStorageField = linker.linkStorageField; + const collection = childCollectionNode.collection; + + if (collection.firewall) { + collection.firewall(filters, options, userId); + } + + filters = cleanUndefinedLeafs(filters); + + const dataProjection = {}; + _.each(options.fields, (value, field) => { + dataProjection[field] = 1; + }); + + if (!isFieldInProjection(dataProjection, linkStorageField, true)) { + dataProjection[linkStorageField] = 1; + } + + function cleanUndefinedLeafs(tree) { + const a = Object.assign({}, tree); + _.each(a, (value, key) => { + if (value === undefined) { + delete a[key]; + } + + if (!_.isArray(value) && _.isObject(value) && !(value instanceof Date)) { + a[key] = cleanUndefinedLeafs(value); + } + }); + + return a; + } + + return {filters, options: {...options, fields: dataProjection}}; +} diff --git a/lib/query/hypernova/processVirtualNode.js b/lib/query/hypernova/processVirtualNode.js new file mode 100644 index 00000000..f61bbf8f --- /dev/null +++ b/lib/query/hypernova/processVirtualNode.js @@ -0,0 +1,99 @@ +import sift from 'sift'; +import dot from 'dot-object'; +import cleanObjectForMetaFilters from './lib/cleanObjectForMetaFilters'; + +function getSlicedResults(results, limit, skip) { + if (_.isFinite(limit) || _.isFinite(skip)) { + skip = skip || 0; + return results.slice(skip, !_.isFinite(limit) ? undefined : skip + limit); + } + return results; +} + +/** + * This only applies to inversed links. It will assemble the data in a correct manner. + */ +export default function(childCollectionNode, results, metaFilters, options = {}) { + const {limit, skip} = options; + const linker = childCollectionNode.linker; + const linkStorageField = linker.linkStorageField; + const linkField = linkStorageField + (linker.isMeta() ? '._id' : ''); + const linkName = childCollectionNode.linkName; + const isMeta = linker.isMeta(); + const isMany = linker.isMany(); + + let allResults = []; + + if (isMeta && metaFilters) { + const metaFiltersTest = sift(metaFilters); + _.each(childCollectionNode.parent.results, parentResult => { + cleanObjectForMetaFilters( + parentResult, + linkStorageField, + metaFiltersTest + ); + }); + } + + if (isMeta && isMany) { + // This case is treated differently because we get an array response from the pipeline. + + _.each(childCollectionNode.parent.results, parentResult => { + parentResult[linkName] = parentResult[linkName] || []; + + const eligibleResults = _.filter( + results, + result => { + // console.log('testing', parentResult._id, (result[linkStorageField] || []).map(value => value._id)); + return _.contains((result[linkStorageField] || []).map(value => value._id), parentResult._id); + }, + ); + + if (eligibleResults.length) { + parentResult[linkName].push(...getSlicedResults(eligibleResults, limit, skip)); + } + }); + + _.each(results, result => { + allResults.push(result); + }); + } else { + let comparator; + if (isMany) { + comparator = (result, parent) => { + const [root, ...nestedFields] = linkField.split('.'); + if (nestedFields.length > 0) { + return _.contains((result[root] || []).map(nestedObject => dot.pick(nestedFields.join('.'), nestedObject)), parent._id); + } + return _.contains(result[linkField], parent._id); + }; + } else { + comparator = (result, parent) => + dot.pick(linkField, result) == parent._id; + } + + const childLinkName = childCollectionNode.linkName; + const parentResults = childCollectionNode.parent.results; + + parentResults.forEach(parentResult => { + // We are now finding the data from the pipeline that is related to the _id of the parent + const eligibleResults = results.filter( + result => comparator(result, parentResult) + ); + + getSlicedResults(eligibleResults, limit, skip).forEach(result => { + if (Array.isArray(parentResult[childLinkName])) { + parentResult[childLinkName].push(result); + } else { + parentResult[childLinkName] = [result]; + } + }); + }); + + results.forEach(result => { + allResults.push(result); + }); + } + + childCollectionNode.results = allResults; +} diff --git a/lib/query/hypernova/storeHypernovaResults.js b/lib/query/hypernova/storeHypernovaResults.js index b2b5945b..034e63df 100755 --- a/lib/query/hypernova/storeHypernovaResults.js +++ b/lib/query/hypernova/storeHypernovaResults.js @@ -1,9 +1,8 @@ import applyProps from '../lib/applyProps.js'; import AggregateFilters from './aggregateSearchFilters.js'; import assemble from './assembler.js'; -import assembleAggregateResults from './assembleAggregateResults.js'; -import buildAggregatePipeline from './buildAggregatePipeline.js'; -import snapBackDottedFields from './lib/snapBackDottedFields'; +import processVirtualNode from './processVirtualNode.js'; +import buildVirtualNodeProps from './buildVirtualNodeProps.js'; export default function storeHypernovaResults(childCollectionNode, userId) { if (childCollectionNode.parent.results.length === 0) { @@ -23,7 +22,6 @@ export default function storeHypernovaResults(childCollectionNode, userId) { const isVirtual = linker.isVirtual(); const collection = childCollectionNode.collection; - _.extend(filters, aggregateFilters.create()); // if it's not virtual then we retrieve them and assemble them here. @@ -40,27 +38,26 @@ export default function storeHypernovaResults(childCollectionNode, userId) { }); } else { // virtuals arrive here - let { pipeline, containsDottedFields } = buildAggregatePipeline( + const virtualProps = buildVirtualNodeProps( childCollectionNode, filters, options, userId ); - let aggregateResults = collection.aggregate(pipeline); + const {filters: virtualFilters, options: {limit, skip, ...virtualOptions}} = virtualProps; + + // console.log(JSON.stringify(virtualProps, null, 4)); + + const results = collection.find(virtualFilters, virtualOptions).fetch(); - /** - * If in aggregation it contains '.', we replace it with a custom string '___' - * And then after aggregation is complete we need to snap-it back to how it was. - */ - if (containsDottedFields) { - snapBackDottedFields(aggregateResults); - } + // console.log(JSON.stringify(results, null, 4)); - assembleAggregateResults( + processVirtualNode( childCollectionNode, - aggregateResults, - metaFilters + results, + metaFilters, + {limit, skip}, ); } } diff --git a/lib/query/testing/server.test.js b/lib/query/testing/server.test.js index 8605272f..2c89eef4 100755 --- a/lib/query/testing/server.test.js +++ b/lib/query/testing/server.test.js @@ -1505,4 +1505,128 @@ describe("intersectDeep", () => { }).fetchOne(); expect(b3.a.length).to.equal(3); // This returns 0, but should be 3 }); + + it("It should work with reverted link and objects inside array", () => { + const A = new Mongo.Collection(Random.id()); + const B = new Mongo.Collection(Random.id()); + A.addLinks({ + b: { + field: "bId", + collection: B, + } + }); + + B.addLinks({ + a: { + collection: A, + inversedBy: "b", + } + }); + const bId = B.insert({}); + + const aId1 = A.insert({ + _id: "aId1", + title: "A1", + bId, + ratings: [ + { + rating: 1, + dimension: '1' + }, + { + rating: 2, + dimension: '2' + } + ] + }); + + const bObj = B.createQuery({ + a: { + ratings: { + rating: 1, + dimension: 1, + } + } + }).fetchOne(); + + assert.isObject(bObj); + assert.isArray(bObj.a); + assert.isArray(bObj.a[0].ratings); + + + bObj.a[0].ratings.forEach(r => { + assert.isString(r.dimension); + assert.isNumber(r.rating); + }) + }); +}); + +describe('collection transforms', () => { + const TransformA = new Mongo.Collection('transformA', { transform: doc => ({ ...doc, hello: 'world1' }) }); + const TransformB = new Mongo.Collection('transformB', { transform:doc => ({ ...doc, hello: 'world2' }) }); + + TransformA.addLinks({ + b: { + collection: TransformB, + field: 'bLink', + type: 'one' + } + }); + + TransformB.addLinks({ + a: { + collection: TransformA, + inversedBy: 'b' + } + }) + + beforeEach(() => { + TransformA.remove({}); + TransformB.remove({}); + const b = TransformB.insert({ value: 1 }); + const a1 = TransformA.insert({ value: 1, bLink: b }); + const a2 = TransformA.insert({ value: 2, bLink: b }); + }) + + it('Should apply transforms on direct side', () => { + const result = createQuery({ + transformB: { + value: 1, + a: { value: 1 } + } + }).fetchOne() + + expect(result.hello).to.equal('world2') + expect(result.a[0].hello).to.equal('world1') + expect(result.a[1].hello).to.equal('world1') + }) + + it('Should apply transforms on indirect side', () => { + const [result1, result2] = createQuery({ + transformA: { + value: 1, + b: { value: 1 } + } + }).fetch() + + expect(result1.hello).to.equal('world1') + expect(result1.b.hello).to.equal('world2') + expect(result2.hello).to.equal('world1') + expect(result2.b.hello).to.equal('world2') + }) + + it('Should apply transforms in a nested way', () => { + const result = createQuery({ + transformA: { + value: 1, + b: { value: 1, a: { value: 1, b: { value: 1 } } } + } + }).fetchOne() + + expect(result.hello).to.equal('world1') + expect(result.b.hello).to.equal('world2') + expect(result.b.a[0].hello).to.equal('world1') + expect(result.b.a[1].hello).to.equal('world1') + expect(result.b.a[0].b.hello).to.equal('world2') + }) }); From b4656c02ae5a95575a69156d03f3e6e94a5b66fe Mon Sep 17 00:00:00 2001 From: Berislav Date: Tue, 24 Oct 2023 15:34:52 +0200 Subject: [PATCH 11/14] Support for foreignField in links --- .gitignore | 3 +- README.md | 19 +++ lib/links/config.schema.js | 3 +- lib/links/lib/createSearchFilters.js | 41 ++++--- lib/links/linkTypes/base.js | 14 ++- lib/links/linkTypes/lib/smartArguments.js | 10 +- lib/links/linkTypes/linkMany.js | 23 ++-- lib/links/linker.js | 23 +++- lib/links/tests/main.js | 109 ++++++++++++++++++ .../testing/bootstrap/queries/index.js | 7 +- .../queries/productAttributesList.js | 19 +++ .../testing/bootstrap/queries/productsList.js | 23 ++++ lib/namedQuery/testing/client.test.js | 68 ++++++++++- lib/namedQuery/testing/server.test.js | 34 ++++++ lib/query/hypernova/aggregateSearchFilters.js | 8 +- lib/query/hypernova/assembler.js | 7 +- lib/query/hypernova/processVirtualNode.js | 8 +- lib/query/hypernova/storeHypernovaResults.js | 4 +- lib/query/nodes/collectionNode.js | 4 + lib/query/testing/bootstrap/fixtures.js | 44 +++++++ lib/query/testing/bootstrap/index.js | 4 + .../testing/bootstrap/products/collection.js | 2 + lib/query/testing/bootstrap/products/links.js | 29 +++++ lib/query/testing/client.test.js | 96 +++++++++++++++ 24 files changed, 548 insertions(+), 54 deletions(-) mode change 100755 => 100644 lib/links/linkTypes/linkMany.js mode change 100755 => 100644 lib/links/tests/main.js create mode 100644 lib/namedQuery/testing/bootstrap/queries/productAttributesList.js create mode 100644 lib/namedQuery/testing/bootstrap/queries/productsList.js mode change 100755 => 100644 lib/query/hypernova/aggregateSearchFilters.js mode change 100755 => 100644 lib/query/hypernova/assembler.js mode change 100755 => 100644 lib/query/hypernova/storeHypernovaResults.js mode change 100755 => 100644 lib/query/nodes/collectionNode.js create mode 100644 lib/query/testing/bootstrap/products/collection.js create mode 100644 lib/query/testing/bootstrap/products/links.js diff --git a/.gitignore b/.gitignore index 4e9f1a0a..0c1d6984 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .DS_Store .idea/ -node_modules/ \ No newline at end of file +node_modules/ +.vscode/ \ No newline at end of file diff --git a/README.md b/README.md index 906c2ac4..3c9fe2f0 100644 --- a/README.md +++ b/README.md @@ -129,3 +129,22 @@ Result: } ] ``` + +### Testing + +You can create test directory and configure dependencies like this (working directory is the root of this repo): +``` +# create meteor app for testing +meteor create --release 1.8.1-rc.1 --bare test +cd test +meteor npm i --save selenium-webdriver@3.6.0 chromedriver@2.36.0 simpl-schema chai + +# Running tests (always from test directory) +METEOR_PACKAGE_DIRS="../" TEST_BROWSER_DRIVER=chrome meteor test-packages --once --driver-package meteortesting:mocha ../ +``` + +If you use `TEST_BROWSER_DRIVER=chrome` you have to have chrome installed in the test environment. Otherwise, you can just run tests in your browsers. + +With `--port=X` you can run tests on port X. + +Omit `--once` and mocha will run in watch mode. diff --git a/lib/links/config.schema.js b/lib/links/config.schema.js index ca2390a7..467afb3f 100644 --- a/lib/links/config.schema.js +++ b/lib/links/config.schema.js @@ -24,6 +24,7 @@ export const LinkConfigSchema = { ); }) ), + foreignIdentityField: Match.Maybe(String), field: Match.Maybe(String), metadata: Match.Maybe(Boolean), inversedBy: Match.Maybe(String), @@ -31,4 +32,4 @@ export const LinkConfigSchema = { unique: Match.Maybe(Boolean), autoremove: Match.Maybe(Boolean), denormalize: Match.Maybe(Match.ObjectIncluding(DenormalizeSchema)), -}; \ No newline at end of file +}; diff --git a/lib/links/lib/createSearchFilters.js b/lib/links/lib/createSearchFilters.js index c1237746..7a01e4e5 100755 --- a/lib/links/lib/createSearchFilters.js +++ b/lib/links/lib/createSearchFilters.js @@ -1,21 +1,24 @@ import sift from 'sift'; import dot from 'dot-object'; -export default function createSearchFilters(object, fieldStorage, strategy, isVirtual, metaFilters) { - if (!isVirtual) { +export default function createSearchFilters(object, linker, metaFilters) { + const fieldStorage = linker.linkStorageField; + + const strategy = linker.strategy; + if (!linker.isVirtual()) { switch (strategy) { - case 'one': return createOne(object, fieldStorage); + case 'one': return createOne(object, linker); case 'one-meta': return createOneMeta(object, fieldStorage, metaFilters); - case 'many': return createMany(object, fieldStorage); + case 'many': return createMany(object, linker); case 'many-meta': return createManyMeta(object, fieldStorage, metaFilters); default: throw new Meteor.Error(`Invalid linking strategy: ${strategy}`) } } else { switch (strategy) { - case 'one': return createOneVirtual(object, fieldStorage); + case 'one': return createOneVirtual(object, linker); case 'one-meta': return createOneMetaVirtual(object, fieldStorage, metaFilters); - case 'many': return createManyVirtual(object, fieldStorage); + case 'many': return createManyVirtual(object, linker); case 'many-meta': return createManyMetaVirtual(object, fieldStorage, metaFilters); default: throw new Meteor.Error(`Invalid linking strategy: ${strategy}`) @@ -23,15 +26,18 @@ export default function createSearchFilters(object, fieldStorage, strategy, isVi } } -export function createOne(object, fieldStorage) { +export function createOne(object, linker) { return { - _id: dot.pick(fieldStorage, object) + // Using {$in: []} as a workaround because foreignIdentityField which is not _id is not required to be set + // and {something: undefined} in query returns all the records. + // $in: [] ensures that nothing will be returned for this query + [linker.foreignIdentityField]: dot.pick(linker.linkStorageField, object) || {$in: []}, }; } -export function createOneVirtual(object, fieldStorage) { +export function createOneVirtual(object, linker) { return { - [fieldStorage]: object._id + [linker.linkStorageField]: object[linker.foreignIdentityField] || {$in: []} }; } @@ -62,25 +68,26 @@ export function createOneMetaVirtual(object, fieldStorage, metaFilters) { return filters; } -export function createMany(object, fieldStorage) { - const [root, ...nested] = fieldStorage.split('.'); +export function createMany(object, linker) { + const [root, ...nested] = linker.linkStorageField.split('.'); if (nested.length > 0) { const arr = object[root]; const ids = arr ? _.uniq(_.union(arr.map(obj => _.isObject(obj) ? dot.pick(nested.join('.'), obj) : []))) : []; return { - _id: {$in: ids} + [linker.foreignIdentityField]: {$in: ids} }; } + const value = object[linker.linkStorageField]; return { - _id: { - $in: object[fieldStorage] || [] + [linker.foreignIdentityField]: { + $in: _.isArray(value) ? value : (value ? [value] : []), } }; } -export function createManyVirtual(object, fieldStorage) { +export function createManyVirtual(object, linker) { return { - [fieldStorage]: object._id + [linker.linkStorageField]: object[linker.foreignIdentityField] || {$in: []}, }; } diff --git a/lib/links/linkTypes/base.js b/lib/links/linkTypes/base.js index 1bc3425b..4c57e1b4 100644 --- a/lib/links/linkTypes/base.js +++ b/lib/links/linkTypes/base.js @@ -50,14 +50,14 @@ export default class Link { const searchFilters = createSearchFilters( this.object, - this.linkStorageField, - linker.strategy, - linker.isVirtual(), + this.linker, $metaFilters ); let appliedFilters = _.extend({}, filters, searchFilters); + // console.log('search filters', searchFilters); + // see https://github.com/cult-of-coders/grapher/issues/134 // happens due to recursive importing of modules // TODO: find another way to do this @@ -133,9 +133,13 @@ export default class Link { ids = [ids]; } + // console.log('validate ids', ids); + + const foreignIdentityField = this.linker.foreignIdentityField; + const validIds = this.linkedCollection.find({ - _id: {$in: ids} - }, {fields: {_id: 1}}).fetch().map(doc => doc._id); + [foreignIdentityField]: {$in: ids} + }, {fields: {[foreignIdentityField]: 1}}).fetch().map(doc => doc[foreignIdentityField]); if (validIds.length != ids.length) { throw new Meteor.Error('not-found', `You tried to create links with non-existing id(s) inside "${this.linkedCollection._name}": ${_.difference(ids, validIds).join(', ')}`) diff --git a/lib/links/linkTypes/lib/smartArguments.js b/lib/links/linkTypes/lib/smartArguments.js index 9c4df47c..b1fdf7fa 100644 --- a/lib/links/linkTypes/lib/smartArguments.js +++ b/lib/links/linkTypes/lib/smartArguments.js @@ -7,21 +7,21 @@ export default new class { getIds(what, options) { if (Array.isArray(what)) { return _.map(what, (subWhat) => { - return this.getId(subWhat, options) - }) + return this.getId(subWhat, options); + }).filter(id => _.isString(id)); } else { - return [this.getId(what, options)]; + return [this.getId(what, options)].filter(id => _.isString(id)); } throw new Meteor.Error('invalid-type', `Unrecognized type: ${typeof what} for managing links`); } getId(what, options) { - if (typeof what === 'string') { + if (_.isString(what)) { return what; } - if (typeof what === 'object') { + if (_.isObject(what)) { if (!what._id && options.saveToDatabase) { what._id = options.collection.insert(what); } diff --git a/lib/links/linkTypes/linkMany.js b/lib/links/linkTypes/linkMany.js old mode 100755 new mode 100644 index b3a5af5d..681d0f61 --- a/lib/links/linkTypes/linkMany.js +++ b/lib/links/linkTypes/linkMany.js @@ -59,6 +59,7 @@ export default class LinkMany extends Link { if (this.isVirtual) throw new Meteor.Error('not-allowed', 'Add/Remove operations should be done from the owner of the relationship'); this.clean(); + const field = this.linkStorageField; const [root, ...nested] = field.split('.'); @@ -67,15 +68,23 @@ export default class LinkMany extends Link { // update the field this.object[root] = _.filter( this.object[root], - _id => !_.contains(_ids, nested.length > 0 ? dot.pick(nested.join('.'), _id) : _id) + _id => !_.contains(_ids, nested.length > 0 ? dot.pick(nested.join('.'), _id) : _id) ); - // update the db - let modifier = { - $pullAll: { - [root]: nested.length > 0 ? { [nested.join('.')]: _ids } : _ids, - }, - }; + let modifier; + if (this.linker.foreignIdentityField === '_id') { + // update the db + modifier = { + $pullAll: { + [root]: nested.length > 0 ? { [nested.join('.')]: _ids } : _ids, + }, + }; + } + else { + modifier = { + $unset: {[root]: 1}, + }; + } this.linker.mainCollection.update(this.object._id, modifier); diff --git a/lib/links/linker.js b/lib/links/linker.js index a6ec53a8..5699c49e 100644 --- a/lib/links/linker.js +++ b/lib/links/linker.js @@ -70,6 +70,16 @@ export default class Linker { return this.linkConfig.field; } + /** + * Returns foreign field for querying linked collection + */ + get foreignIdentityField() { + if (this.isVirtual()) { + return this.linkConfig.relatedLinker.linkConfig.foreignIdentityField || '_id'; + } + return this.linkConfig.foreignIdentityField || '_id'; + } + /** * The collection that is linked with the current collection * @returns Mongo.Collection @@ -377,11 +387,14 @@ export default class Linker { if (!this.isVirtual()) { this.mainCollection.after.remove((userId, doc) => { - this.getLinkedCollection().remove({ - _id: { - $in: smartArguments.getIds(doc[this.linkStorageField]), - }, - }); + const ids = smartArguments.getIds(doc[this.linkStorageField]); + if (ids.length > 0) { + this.getLinkedCollection().remove({ + [this.foreignIdentityField]: { + $in: ids, + }, + }); + } }); } else { this.mainCollection.after.remove((userId, doc) => { diff --git a/lib/links/tests/main.js b/lib/links/tests/main.js old mode 100755 new mode 100644 index b63d7446..7eddd793 --- a/lib/links/tests/main.js +++ b/lib/links/tests/main.js @@ -11,6 +11,8 @@ let PostCollection = new Mongo.Collection("test_post"); let CategoryCollection = new Mongo.Collection("test_category"); let CommentCollection = new Mongo.Collection("test_comment"); let ResolverCollection = new Mongo.Collection("test_resolver"); +let SCDCollection = new Mongo.Collection('test_scd'); +let ReferenceCollection = new Mongo.Collection('test_scd_refs'); PostCollection.addLinks({ comments: { @@ -91,6 +93,34 @@ CategoryCollection.addLinks({ } }); +ReferenceCollection.addLinks({ + scds: { + type: 'many', + collection: SCDCollection, + field: 'scdId', + foreignIdentityField: 'originalId', + }, + scd: { + collection: SCDCollection, + inversedBy: 'ref', + }, +}) + +SCDCollection.addLinks({ + refs: { + collection: ReferenceCollection, + inversedBy: 'scds', + autoremove: true, + }, + ref: { + type: 'one', + collection: ReferenceCollection, + field: 'someId', + foreignIdentityField: 'some2Id', + autoremove: true, + }, +}); + describe("Collection Links", function() { PostCollection.remove({}); CategoryCollection.remove({}); @@ -269,6 +299,8 @@ describe("Collection Links", function() { assert.isObject(postUniqueLink.fetch()); assert.lengthOf(postMetaLink.find().fetch(), 1); + post = PostCollection.findOne(postId); + CommentCollection.remove(comment._id); post = PostCollection.findOne(postId); assert.notInclude(post.commentIds, comment._id); @@ -456,4 +488,81 @@ describe("Collection Links", function() { PostCollection.remove(postIdB); }); + + describe('foreignIdentityField linkConfig param', function () { + beforeEach(function () { + SCDCollection.remove({}); + ReferenceCollection.remove({}); + }); + + it("Works with foreign field - many", function () { + SCDCollection.insert({_id: '1', originalId: '1'}); + SCDCollection.insert({_id: '2', originalId: '1'}); + SCDCollection.insert({_id: '3', originalId: '3'}); + const scd4Id = SCDCollection.insert({_id: '4', originalId: '4'}); + + ReferenceCollection.insert({scdId: '1'}); + ReferenceCollection.insert({scdId: '3'}); + const ref3Id = ReferenceCollection.insert({}); + + const linkRef = ReferenceCollection.getLink({scdId: '1'}, "scds"); + // both SCDs should be found since they share originalId + assert.lengthOf(linkRef.find().fetch(), 2); + + const linkSCD = SCDCollection.getLink({_id: '2', originalId: '1'}, "refs"); + assert.lengthOf(linkSCD.find().fetch(), 1); + + // check if it works when links do not exist + const link = ReferenceCollection.getLink(ref3Id, "scds"); + assert.lengthOf(link.find().fetch(), 0); + + const inversedLink = SCDCollection.getLink(scd4Id, "refs"); + assert.lengthOf(inversedLink.find().fetch(), 0); + }); + + it("Auto-removes for foreign field - many", function () { + SCDCollection.insert({_id: '1', originalId: '1'}); + SCDCollection.insert({_id: '2', originalId: '1'}); + + ReferenceCollection.insert({scdId: '1'}); + + // assert.equal(ReferenceCollection.find().count(), 0); + SCDCollection.remove('1'); + + assert.equal(ReferenceCollection.find().count(), 0); + }); + + it("Works with foreign field - one", function () { + SCDCollection.insert({someId: '1'}); + + ReferenceCollection.insert({some2Id: '1'}); + const ref2Id = ReferenceCollection.insert({some2Id: '2'}); + + const linkSCD = SCDCollection.getLink({someId: '1'}, "ref"); + assert.lengthOf(linkSCD.find().fetch(), 1); + + const linkRef = ReferenceCollection.getLink({some2Id: '1'}, "scd"); + assert.lengthOf(linkRef.find().fetch(), 1); + + // check if it works when links do not exist + const newId = SCDCollection.insert({}); // no someId + const link = SCDCollection.getLink(newId, "ref"); + assert.lengthOf(link.find().fetch(), 0); + + // inversed + const inversedLink = ReferenceCollection.getLink(ref2Id, "scd"); + assert.lengthOf(inversedLink.find().fetch(), 0); + }); + + it("Auto-removes for foreign field - one", function () { + SCDCollection.insert({_id: '1', someId: '1'}); + + ReferenceCollection.insert({some2Id: '1'}); + ReferenceCollection.insert({some2Id: '2'}); + + SCDCollection.remove('1'); + + assert.equal(ReferenceCollection.find().count(), 1); + }); + }); }); diff --git a/lib/namedQuery/testing/bootstrap/queries/index.js b/lib/namedQuery/testing/bootstrap/queries/index.js index ad35e81d..aef9d050 100755 --- a/lib/namedQuery/testing/bootstrap/queries/index.js +++ b/lib/namedQuery/testing/bootstrap/queries/index.js @@ -6,7 +6,9 @@ import postListParamsCheck from './postListParamsCheck'; import postListParamsCheckServer from './postListParamsCheckServer'; import postListResolver from './postListResolver'; import postListResolverCached from './postListResolverCached'; +import productsList from './productsList'; import userListScoped from './userListScoped'; +import productAttributesList from './productAttributesList'; export { postList, @@ -16,5 +18,8 @@ export { postListParamsCheck, postListParamsCheckServer, postListResolver, - postListResolverCached + postListResolverCached, + userListScoped, + productsList, + productAttributesList, } diff --git a/lib/namedQuery/testing/bootstrap/queries/productAttributesList.js b/lib/namedQuery/testing/bootstrap/queries/productAttributesList.js new file mode 100644 index 00000000..872b4f57 --- /dev/null +++ b/lib/namedQuery/testing/bootstrap/queries/productAttributesList.js @@ -0,0 +1,19 @@ +import { ProductAttributes } from "../../../../query/testing/bootstrap/products/collection"; + +const productAttributesList = ProductAttributes.createQuery('productAttributesList', { + unit: 1, + delivery: 1, + productId: 1, + products: { + title: 1, + productId: 1, + }, +}); + +if (Meteor.isServer) { + productAttributesList.expose({ + firewall() {}, + }); +} + +export default productAttributesList; diff --git a/lib/namedQuery/testing/bootstrap/queries/productsList.js b/lib/namedQuery/testing/bootstrap/queries/productsList.js new file mode 100644 index 00000000..4e44378b --- /dev/null +++ b/lib/namedQuery/testing/bootstrap/queries/productsList.js @@ -0,0 +1,23 @@ +import { Products } from "../../../../query/testing/bootstrap/products/collection"; + +const productsList = Products.createQuery('productsList', { + title: 1, + price: 1, + productId: 1, + attributes: { + productId: 1, + unit: 1, + delivery: 1, + }, + singleAttribute: { + delivery: 1, + }, +}); + +if (Meteor.isServer) { + productsList.expose({ + firewall() {}, + }); +} + +export default productsList; diff --git a/lib/namedQuery/testing/client.test.js b/lib/namedQuery/testing/client.test.js index d40b266f..076437f9 100755 --- a/lib/namedQuery/testing/client.test.js +++ b/lib/namedQuery/testing/client.test.js @@ -1,7 +1,8 @@ -import { assert } from 'chai'; +import { assert, expect } from 'chai'; import postListExposure, {postListFilteredWithDate} from './bootstrap/queries/postListExposure.js'; import postListExposureScoped from './bootstrap/queries/postListExposureScoped'; import userListScoped from './bootstrap/queries/userListScoped'; +import productsList from './bootstrap/queries/productsList'; import { createQuery } from 'meteor/cultofcoders:grapher'; import Posts from '../../query/testing/bootstrap/posts/collection'; import Users from '../../query/testing/bootstrap/users/collection'; @@ -262,4 +263,69 @@ describe('Named Query', function () { } }); }); + + it('Should work with reactive queries containing link with foreignIdentityField', function (done) { + const query = productsList.clone({ + filters: { + // only considering products with productId + productId: {$ne: null}, + } + }); + + const handle = query.subscribe(); + + Tracker.autorun(c => { + if (handle.ready()) { + c.stop(); + const res = query.fetch(); + handle.stop(); + + assert.equal(res.length, 3); + + const [nails1, nails2, laptop] = res; + assert.equal(nails1.price, 1.50); + assert.lengthOf(nails1.attributes, 1); + assert.deepEqual(nails1.attributes[0].unit, 'piece'); + assert.deepEqual(nails1.attributes[0].delivery, 0); + + assert.equal(nails2.price, 1.60); + assert.lengthOf(nails2.attributes, 1); + assert.equal(nails2.attributes[0].unit, 'piece'); + assert.equal(nails2.attributes[0].delivery, 0); + + assert.equal(laptop.price, 1500); + assert.lengthOf(laptop.attributes, 1); + assert.equal(laptop.attributes[0].delivery, 10); + + done(); + } + }); + }); + + it('should work with reactive queries ', (done) => { + const query = productsList.clone({ + filters: { + // only considering products with productId + singleProductId: {$ne: null}, + } + }); + + const handle = query.subscribe(); + + Tracker.autorun(c => { + if (handle.ready()) { + c.stop(); + const res = query.fetch(); + handle.stop(); + + assert.equal(res.length, 1); + + const [laptop] = res; + assert.isObject(laptop.singleAttribute); + assert.equal(laptop.singleAttribute.delivery, 12); + + done(); + } + }); + }); }); diff --git a/lib/namedQuery/testing/server.test.js b/lib/namedQuery/testing/server.test.js index ce61d0c5..117bd9dd 100755 --- a/lib/namedQuery/testing/server.test.js +++ b/lib/namedQuery/testing/server.test.js @@ -6,6 +6,8 @@ import { postListResolverCached, postListParamsCheck, postListParamsCheckServer, + productsList, + productAttributesList, } from './bootstrap/queries'; import { createQuery, NamedQuery } from 'meteor/cultofcoders:grapher'; @@ -175,4 +177,36 @@ describe('Named Query', function () { NamedQuery.setConfig({}); } }); + + it('Should work with foreign field - regular link assembly', () => { + const res = productAttributesList.clone().fetch(); + assert.lengthOf(res, 3); + + res.forEach(attribute => { + if (typeof attribute.productId === 'number') { + assert.isArray(attribute.products); + assert.isTrue(attribute.products.length > 0); + attribute.products.forEach(product => { + assert.equal(product.productId, attribute.productId); + }); + } + else { + assert.isUndefined(attribute.products); + } + }); + }); + + it('Should work with foreign field - inversed link assembly', () => { + const res = productsList.clone({}).fetch(); + + assert.lengthOf(res, 4); + + res.forEach(product => { + if ([1, 2].includes(product.productId)) { + assert.isArray(product.attributes); + assert.lengthOf(product.attributes, 1); + assert.equal(product.attributes[0].productId, product.productId); + } + }); + }); }); diff --git a/lib/query/hypernova/aggregateSearchFilters.js b/lib/query/hypernova/aggregateSearchFilters.js old mode 100755 new mode 100644 index 5c73d4b6..653dae36 --- a/lib/query/hypernova/aggregateSearchFilters.js +++ b/lib/query/hypernova/aggregateSearchFilters.js @@ -22,6 +22,10 @@ export default class AggregateFilters { return this.collectionNode.parent.results; } + get foreignIdentityField() { + return this.linker.foreignIdentityField; + } + create() { switch (this.linker.strategy) { case 'one': @@ -99,12 +103,12 @@ export default class AggregateFilters { const [root, ...nested] = this.linkStorageField.split('.'); const arrayOfIds = _.union(...extractIdsFromArray(this.parentObjects, root)); return { - _id: { + [this.foreignIdentityField]: { $in: _.uniq(nested.length > 0 ? extractIdsFromArray(arrayOfIds, nested.join('.')) : arrayOfIds) } }; } else { - const arrayOfIds = _.pluck(this.parentObjects, '_id'); + const arrayOfIds = _.pluck(this.parentObjects, this.foreignIdentityField); return { [this.linkStorageField]: { $in: _.uniq( diff --git a/lib/query/hypernova/assembler.js b/lib/query/hypernova/assembler.js old mode 100755 new mode 100644 index 762b3130..7f805493 --- a/lib/query/hypernova/assembler.js +++ b/lib/query/hypernova/assembler.js @@ -29,7 +29,7 @@ export default (childCollectionNode, { limit, skip, metaFilters }) => { }); } - const resultsByKeyId = _.groupBy(childCollectionNode.results, '_id'); + const resultsByKeyId = _.groupBy(childCollectionNode.results, linker.foreignIdentityField); if (strategy === 'one') { parent.results.forEach(parentResult => { @@ -54,11 +54,10 @@ export default (childCollectionNode, { limit, skip, metaFilters }) => { return; } - const data = []; - value.forEach(v => { + (_.isArray(value) ? value : [value]).forEach(v => { const _id = nested.length > 0 ? dot.pick(nested.join('.'), v) : v; - data.push(_.first(resultsByKeyId[_id])); + data.push(...(resultsByKeyId[_id] || [])); }); parentResult[childCollectionNode.linkName] = filterAssembledData( diff --git a/lib/query/hypernova/processVirtualNode.js b/lib/query/hypernova/processVirtualNode.js index f61bbf8f..2e9979e0 100644 --- a/lib/query/hypernova/processVirtualNode.js +++ b/lib/query/hypernova/processVirtualNode.js @@ -62,14 +62,14 @@ export default function(childCollectionNode, results, metaFilters, options = {}) if (isMany) { comparator = (result, parent) => { const [root, ...nestedFields] = linkField.split('.'); + const rootValue = _.isArray(result[root]) ? result[root] : [result[root]]; if (nestedFields.length > 0) { - return _.contains((result[root] || []).map(nestedObject => dot.pick(nestedFields.join('.'), nestedObject)), parent._id); + return _.contains(rootValue.map(nestedObject => dot.pick(nestedFields.join('.'), nestedObject)), parent[linker.foreignIdentityField]); } - return _.contains(result[linkField], parent._id); + return _.contains(rootValue ?? [], parent[linker.foreignIdentityField]); }; } else { - comparator = (result, parent) => - dot.pick(linkField, result) == parent._id; + comparator = (result, parent) => dot.pick(linkField, result) == parent[linker.foreignIdentityField]; } const childLinkName = childCollectionNode.linkName; diff --git a/lib/query/hypernova/storeHypernovaResults.js b/lib/query/hypernova/storeHypernovaResults.js old mode 100755 new mode 100644 index 034e63df..88e298dd --- a/lib/query/hypernova/storeHypernovaResults.js +++ b/lib/query/hypernova/storeHypernovaResults.js @@ -18,6 +18,8 @@ export default function storeHypernovaResults(childCollectionNode, userId) { ); delete filters.$meta; + + const linker = childCollectionNode.linker; const isVirtual = linker.isVirtual(); const collection = childCollectionNode.collection; @@ -46,7 +48,7 @@ export default function storeHypernovaResults(childCollectionNode, userId) { ); const {filters: virtualFilters, options: {limit, skip, ...virtualOptions}} = virtualProps; - + // console.log(JSON.stringify(virtualProps, null, 4)); const results = collection.find(virtualFilters, virtualOptions).fetch(); diff --git a/lib/query/nodes/collectionNode.js b/lib/query/nodes/collectionNode.js old mode 100755 new mode 100644 index 28609781..b7ab308a --- a/lib/query/nodes/collectionNode.js +++ b/lib/query/nodes/collectionNode.js @@ -58,6 +58,10 @@ export default class CollectionNode { node.isVirtual = linker.isVirtual(); node.isOneResult = linker.isOneResult(); node.shouldCleanStorage = this._shouldCleanStorage(node); + + if (linker.foreignIdentityField !== '_id' && !node.hasField(linker.foreignIdentityField, true)) { + node.add(new FieldNode(linker.foreignIdentityField, 1)); + } } this.nodes.push(node); diff --git a/lib/query/testing/bootstrap/fixtures.js b/lib/query/testing/bootstrap/fixtures.js index ee118e88..bef7b842 100755 --- a/lib/query/testing/bootstrap/fixtures.js +++ b/lib/query/testing/bootstrap/fixtures.js @@ -10,6 +10,7 @@ import Groups from './groups/collection'; import Users from './users/collection'; import {Files} from './files/collection'; import {Projects} from './projects/collection'; +import {Products, ProductAttributes} from './products/collection'; Authors.remove({}); Comments.remove({}); @@ -19,6 +20,8 @@ Groups.remove({}); Users.remove({}); Files.remove({}); Projects.remove({}); +Products.remove({}); +ProductAttributes.remove({}); const AUTHORS = 6; const POST_PER_USER = 6; @@ -130,4 +133,45 @@ Files.insert({ }, }); +/** + * Products here are slowly changing dimensions, with price being the the field which creates new version. + * However, there are also general attributes which do not create versions, such as units & delivery. + */ +Products.insert({ + title: 'Nails', + price: 1.50, + productId: 1, +}); +Products.insert({ + title: 'Nails', + price: 1.60, + productId: 1, +}); +Products.insert({ + title: 'Laptop', + price: 1500, + productId: 2, +}); +ProductAttributes.insert({ + productId: 1, + unit: 'piece', + delivery: 0, +}); +ProductAttributes.insert({ + productId: 2, + delivery: 10, +}); + +// For testing "one" relationship on product +Products.insert({ + title: 'Laptop', + price: 1300, + singleProductId: 1, +}); + +ProductAttributes.insert({ + singleProductId: 1, + delivery: 12, +}); + console.log('[ok] fixtures have been loaded.'); diff --git a/lib/query/testing/bootstrap/index.js b/lib/query/testing/bootstrap/index.js index e44ff034..ab81f20d 100755 --- a/lib/query/testing/bootstrap/index.js +++ b/lib/query/testing/bootstrap/index.js @@ -7,6 +7,7 @@ import './security/links'; import './users/links'; import './files/links'; import './projects/links'; +import './products/links'; import Posts from './posts/collection'; import Groups from './groups/collection'; @@ -14,6 +15,7 @@ import Authors from './authors/collection'; import Users from './users/collection'; import {Files} from './files/collection'; import {Projects} from './projects/collection'; +import {ProductAttributes, Products} from './products/collection'; if (Meteor.isServer) { Posts.expose(); @@ -22,4 +24,6 @@ if (Meteor.isServer) { Users.expose(); Files.expose(); Projects.expose(); + Products.expose(); + ProductAttributes.expose(); } diff --git a/lib/query/testing/bootstrap/products/collection.js b/lib/query/testing/bootstrap/products/collection.js new file mode 100644 index 00000000..c71d6f98 --- /dev/null +++ b/lib/query/testing/bootstrap/products/collection.js @@ -0,0 +1,2 @@ +export const Products = new Mongo.Collection('products'); +export const ProductAttributes = new Mongo.Collection('product_attributes'); diff --git a/lib/query/testing/bootstrap/products/links.js b/lib/query/testing/bootstrap/products/links.js new file mode 100644 index 00000000..bb72ad78 --- /dev/null +++ b/lib/query/testing/bootstrap/products/links.js @@ -0,0 +1,29 @@ +import {Products, ProductAttributes} from './collection'; + +Products.addLinks({ + // here we are assuming manu relationship + attributes: { + collection: ProductAttributes, + inversedBy: 'products', + }, + singleAttribute: { + type: 'one', + collection: ProductAttributes, + field: 'singleProductId', + foreignIdentityField: 'singleProductId', + unique: true, + }, +}); + +ProductAttributes.addLinks({ + products: { + type: 'many', + collection: Products, + field: 'productId', + foreignIdentityField: 'productId', + }, + singleProduct: { + collection: Products, + inversedBy: 'singleAttribute', + }, +}); diff --git a/lib/query/testing/client.test.js b/lib/query/testing/client.test.js index 5d9829e2..907981ee 100755 --- a/lib/query/testing/client.test.js +++ b/lib/query/testing/client.test.js @@ -485,4 +485,100 @@ describe('Query Client Tests', function () { }); }); }); + + it('Should work with foreignIdentityField - one', async function () { + const query = createQuery({ + products: { + $filters: { + singleProductId: 1, + }, + singleAttribute: { + singleProductId: 1, + delivery: 1, + }, + }, + }); + + const handle = query.subscribe(); + await waitForHandleToBeReady(handle); + + const products = query.fetch(); + expect(products).to.have.length(1); + products.forEach((product) => { + expect(product.singleAttribute).to.be.an('object'); + expect(product.singleAttribute.delivery).to.be.equal(12); + expect(product.singleAttribute.singleProductId).to.be.equal(1); + }); + }); + + + it('Should work with foreignIdentityField - one inversed', async function () { + const query = createQuery({ + product_attributes: { + $filters: { + singleProductId: 1, + }, + singleProduct: { + price: 1, + }, + }, + }); + + const handle = query.subscribe(); + await waitForHandleToBeReady(handle); + + const products = query.fetch(); + expect(products).to.have.length(1); + products.forEach((product) => { + expect(product.singleProduct).to.be.an('object'); + expect(product.singleProduct.price).to.be.equal(1300); + }); + }); + + it('Should work with foreignIdentityField - many', async function () { + const query = createQuery({ + product_attributes: { + $filters: { + productId: {$ne: null}, + }, + products: { + _id: 1, + productId: 1, + }, + }, + }); + + const handle = query.subscribe(); + await waitForHandleToBeReady(handle); + + const attributes = query.fetch(); + expect(attributes).to.have.length(2); + // see fixtures.js + expect(attributes[0].products).to.have.length(2); + expect(attributes[1].products).to.have.length(1); + }); + + it('Should work with foreignIdentityField - many inversed', async function () { + const query = createQuery({ + products: { + $filters: { + productId: {$ne: null}, + }, + productId: 1, + attributes: { + _id: 1, + productId: 1, + }, + }, + }); + + const handle = query.subscribe(); + await waitForHandleToBeReady(handle); + + const products = query.fetch(); + expect(products).to.have.length(3); + expect(products[0].attributes).to.have.length(1); + expect(products[1].attributes).to.have.length(1); + expect(products[2].attributes).to.have.length(1); + }); }); From 2784d2dcec80d72ea7fca5a20ff8c7612aefb074 Mon Sep 17 00:00:00 2001 From: Berislav Date: Wed, 25 Oct 2023 17:44:30 +0200 Subject: [PATCH 12/14] Updated docs --- docs/linking_collections.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/linking_collections.md b/docs/linking_collections.md index 7a4266ff..b5731b1b 100644 --- a/docs/linking_collections.md +++ b/docs/linking_collections.md @@ -386,6 +386,43 @@ Meteor.users.createQuery({ `paymentProfile` inside `user` will be an object because it knows it should be unique. +## Foreign identity field + +When you add a link by default grapher tries to match against `_id` of the linked collection. + +Consider a system with two collections: + +- `Appointments` - a collection of appointments with startDate, endDate, etc. +- `Tasks` - a collection of tasks which has `referenceId` field which is `_id` of the appointment or some other entity. Tasks generally don't know anything about the appointment, they just have a reference to it. + +We can utilize `foreignIdentityField` option and do this: + +```js +Appointments.addLinks({ + tasks: { + collection: Tasks, + type: "many", + field: "_id", // field from Appointments collection + foreignIdentityField: "referenceId", // field from Tasks collection + }, +}); +``` + +Now you can query for appointments and get all tasks for each appointment: + +```js +Appointments.createQuery({ + $filters: { ... }, + tasks: { + title: 1, + }, + startDate: 1, + endDate: 1, +}).fetch(); +``` + +If your foreign identity field is unique inside linked collection (in this case Tasks), you can use `type: "one"` and get a single task instead of an array. + ## Data Consistency We clean out leftover links from deleted collection items. From 8c9e1c4596fe6af57ad2346d641729f747ed0461 Mon Sep 17 00:00:00 2001 From: Berislav Date: Wed, 25 Oct 2023 17:50:04 +0200 Subject: [PATCH 13/14] Updated Readme --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3c9fe2f0..63991dc8 100644 --- a/README.md +++ b/README.md @@ -132,18 +132,22 @@ Result: ### Testing -You can create test directory and configure dependencies like this (working directory is the root of this repo): +You can create `test` directory and configure dependencies (working directory is the root of this repo): ``` # create meteor app for testing -meteor create --release 1.8.1-rc.1 --bare test +# you can add a specific release with --release flag, this will just create the app with the latest release +meteor create --bare test cd test -meteor npm i --save selenium-webdriver@3.6.0 chromedriver@2.36.0 simpl-schema chai +# install npm dependencies used for testing +meteor npm i --save selenium-webdriver@3.6.0 chromedriver@2.36.0 simpl-schema@1.13.1 chai -# Running tests (always from test directory) +# Running tests (always from ./test directory) METEOR_PACKAGE_DIRS="../" TEST_BROWSER_DRIVER=chrome meteor test-packages --once --driver-package meteortesting:mocha ../ ``` -If you use `TEST_BROWSER_DRIVER=chrome` you have to have chrome installed in the test environment. Otherwise, you can just run tests in your browsers. +If you use `TEST_BROWSER_DRIVER=chrome` you have to have chrome installed in the test environment. Otherwise, you can just run tests in your browser. + +Another option is to use `puppeteer` as a driver. You'll have to install it with `meteor npm i puppeteer@10`. Note that the latest versions don't work with Node 14. With `--port=X` you can run tests on port X. From fb271e994ee473c6e448c7741cffb71da6f050e3 Mon Sep 17 00:00:00 2001 From: Berislav Date: Wed, 25 Oct 2023 17:52:50 +0200 Subject: [PATCH 14/14] Get rid of _.isString --- lib/links/linkTypes/lib/smartArguments.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/links/linkTypes/lib/smartArguments.js b/lib/links/linkTypes/lib/smartArguments.js index b1fdf7fa..5bf0200d 100644 --- a/lib/links/linkTypes/lib/smartArguments.js +++ b/lib/links/linkTypes/lib/smartArguments.js @@ -8,16 +8,14 @@ export default new class { if (Array.isArray(what)) { return _.map(what, (subWhat) => { return this.getId(subWhat, options); - }).filter(id => _.isString(id)); + }).filter(id => typeof id === 'string'); } else { - return [this.getId(what, options)].filter(id => _.isString(id)); + return [this.getId(what, options)].filter(id => typeof id === 'string'); } - - throw new Meteor.Error('invalid-type', `Unrecognized type: ${typeof what} for managing links`); } getId(what, options) { - if (_.isString(what)) { + if (typeof what === 'string') { return what; }