From 192a7db8faca5894780bb846a37989163a2a64af Mon Sep 17 00:00:00 2001 From: "nick.tessier" <22119573+nick4598@users.noreply.github.com> Date: Tue, 7 Jan 2025 11:58:41 -0500 Subject: [PATCH 1/4] wip test is failing --- .../IModelBranchingOperations.test.ts | 302 ++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100644 packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts diff --git a/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts new file mode 100644 index 00000000..0001046d --- /dev/null +++ b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts @@ -0,0 +1,302 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as path from "path"; +import * as TestUtils from "../TestUtils"; +import { + BriefcaseDb, + BriefcaseManager, + ExternalSource, + ExternalSourceIsInRepository, + HubMock, + IModelDb, + IModelHost, + PhysicalModel, + PhysicalObject, + PhysicalPartition, + RepositoryLink, + SnapshotDb, + SpatialCategory, +} from "@itwin/core-backend"; +import { + HubWrappers, + IModelTransformerTestUtils, +} from "../IModelTransformerUtils"; +import { AccessToken } from "@itwin/core-bentley"; +import { + Code, + ExternalSourceProps, + IModel, + PhysicalElementProps, + RepositoryLinkProps, + SubCategoryAppearance, +} from "@itwin/core-common"; +import { Point3d, YawPitchRollAngles } from "@itwin/core-geometry"; +import { IModelTransformer } from "../../IModelTransformer"; +process.env.TRANSFORMER_NO_STRICT_DEP_CHECK = "1"; // allow this monorepo's dev versions of core libs in transformer + +// some json will be required later, but we don't want an eslint-disable line in the example code, so just disable for the file +/* eslint-disable @typescript-eslint/no-require-imports */ + +async function initializeBranch( + myITwinId: string, + masterIModelId: string, + myAccessToken: AccessToken +) { + // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_initialize + // download and open master + const masterDbProps = await BriefcaseManager.downloadBriefcase({ + accessToken: myAccessToken, + iTwinId: myITwinId, + iModelId: masterIModelId, + }); + const masterDb = await BriefcaseDb.open({ fileName: masterDbProps.fileName }); + + // create a duplicate of master as a good starting point for our branch + const branchIModelId = await IModelHost.hubAccess.createNewIModel({ + iTwinId: myITwinId, + iModelName: "my-branch-imodel", + version0: masterDb.pathName, + noLocks: true, // you may prefer locks for your application + }); + + // download and open the new branch + const branchDbProps = await BriefcaseManager.downloadBriefcase({ + accessToken: myAccessToken, + iTwinId: myITwinId, + iModelId: branchIModelId, + }); + const branchDb = await BriefcaseDb.open({ fileName: branchDbProps.fileName }); + + // create an external source and owning repository link to use as our *Target Scope Element* for future synchronizations + const masterLinkRepoId = branchDb + .constructEntity({ + classFullName: RepositoryLink.classFullName, + code: RepositoryLink.createCode( + branchDb, + IModelDb.repositoryModelId, + "example-code-value" + ), + model: IModelDb.repositoryModelId, + url: "https://wherever-you-got-your-imodel.net", + format: "iModel", + repositoryGuid: masterDb.iModelId, + description: "master iModel repository", + }) + .insert(); + + const masterExternalSourceId = branchDb + .constructEntity({ + classFullName: ExternalSource.classFullName, + model: IModelDb.rootSubjectId, + code: Code.createEmpty(), + repository: new ExternalSourceIsInRepository(masterLinkRepoId), + connectorName: "iModel Transformer", + connectorVersion: require("@itwin/imodel-transformer/package.json") + .version, + }) + .insert(); + + // initialize the branch provenance + const branchInitializer = new IModelTransformer(masterDb, branchDb, { + // tells the transformer that we have a raw copy of a source and the target should receive + // provenance from the source that is necessary for performing synchronizations in the future + wasSourceIModelCopiedToTarget: true, + // store the synchronization provenance in the scope of our representation of the external source, master + targetScopeElementId: masterExternalSourceId, + }); + await branchInitializer.process(); + branchInitializer.dispose(); + + // save+push our changes to whatever hub we're using + const description = "initialized branch iModel"; + branchDb.saveChanges(description); + await branchDb.pushChanges({ + accessToken: myAccessToken, + description, + }); + // __PUBLISH_EXTRACT_END__ + + return { masterDb, branchDb }; +} + +// we assume masterDb and branchDb have already been opened (see the first example) +async function forwardSyncMasterToBranch( + masterDb: BriefcaseDb, + branchDb: BriefcaseDb, + myAccessToken: AccessToken +) { + // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_forwardSync + const masterExternalSourceId = branchDb.elements.queryElementIdByCode( + RepositoryLink.createCode( + masterDb, + IModelDb.repositoryModelId, + "example-code-value" + ) + ); + const synchronizer = new IModelTransformer(masterDb, branchDb, { + // read the synchronization provenance in the scope of our representation of the external source, master + targetScopeElementId: masterExternalSourceId, + // Presence of argsForProcessChanges even if empty is required to have process run 'processChanges' internally. + argsForProcessChanges: {}, + }); + + await synchronizer.process(); + synchronizer.dispose(); + // save and push + const description = "updated branch with recent master changes"; + branchDb.saveChanges(description); + await branchDb.pushChanges({ + accessToken: myAccessToken, + description, + }); + // __PUBLISH_EXTRACT_END__ +} + +async function reverseSyncBranchToMaster( + branchDb: BriefcaseDb, + masterDb: BriefcaseDb, + myAccessToken: AccessToken +) { + // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_reverseSync + // we assume masterDb and branchDb have already been opened (see the first example) + const masterExternalSourceId = branchDb.elements.queryElementIdByCode( + RepositoryLink.createCode( + masterDb, + IModelDb.repositoryModelId, + "example-code-value" + ) + ); + const reverseSynchronizer = new IModelTransformer(branchDb, masterDb, { + // read the synchronization provenance in the scope of our representation of the external source, master + // "isReverseSynchronization" actually causes the provenance (and therefore the targetScopeElementId) to + // be searched for from the source + targetScopeElementId: masterExternalSourceId, + // Presence of argsForProcessChanges even if empty is required to have process run 'processChanges' internally. + argsForProcessChanges: {}, + }); + + await reverseSynchronizer.process(); + reverseSynchronizer.dispose(); + // save and push + const description = "merged changes from branch into master"; + masterDb.saveChanges(description); + await masterDb.pushChanges({ + accessToken: myAccessToken, + description, + }); + // __PUBLISH_EXTRACT_END__ +} + +async function arbitraryEdit( + db: BriefcaseDb, + myAccessToken: AccessToken, + description: string +) { + const spatialCategoryCode = SpatialCategory.createCode( + db, + IModel.dictionaryId, + "SpatialCategory1" + ); + const physicalModelCode = PhysicalPartition.createCode( + db, + IModel.rootSubjectId, + "PhysicalModel1" + ); + let spatialCategoryId = db.elements.queryElementIdByCode(spatialCategoryCode); + let physicalModelId = db.elements.queryElementIdByCode(physicalModelCode); + if (physicalModelId === undefined || spatialCategoryId === undefined) { + spatialCategoryId = SpatialCategory.insert( + db, + IModel.dictionaryId, + "SpatialCategory1", + new SubCategoryAppearance() + ); + physicalModelId = PhysicalModel.insert( + db, + IModel.rootSubjectId, + "PhysicalModel1" + ); + } + const physicalObjectProps: PhysicalElementProps = { + classFullName: PhysicalObject.classFullName, + model: physicalModelId, + category: spatialCategoryId, + code: new Code({ + spec: IModelDb.rootSubjectId, + scope: IModelDb.rootSubjectId, + value: `${arbitraryEdit.editCounter}`, + }), + userLabel: `${arbitraryEdit.editCounter}`, + geom: IModelTransformerTestUtils.createBox(Point3d.create(1, 1, 1)), + placement: { + origin: Point3d.create( + arbitraryEdit.editCounter, + arbitraryEdit.editCounter, + 0 + ), + angles: YawPitchRollAngles.createDegrees(0, 0, 0), + }, + }; + arbitraryEdit.editCounter++; + db.elements.insertElement(physicalObjectProps); + db.saveChanges(); + await db.pushChanges({ + accessToken: myAccessToken, + description, + }); +} + +namespace arbitraryEdit { + // eslint-disable-next-line prefer-const + export let editCounter = 0; +} + +describe.only("IModelBranchingOperations", () => { + const version0Path = path.join( + TestUtils.KnownTestLocations.outputDir, + "branching-ops.bim" + ); + + before(async () => { + HubMock.startup( + "IModelBranchingOperations", + TestUtils.KnownTestLocations.outputDir + ); + if (fs.existsSync(version0Path)) fs.unlinkSync(version0Path); + SnapshotDb.createEmpty(version0Path, { + rootSubject: { name: "branching-ops" }, + }).close(); + }); + + after(() => { + HubMock.shutdown(); + }); + + it("run branching operations", async () => { + const myAccessToken = await HubWrappers.getAccessToken( + TestUtils.TestUserType.Regular + ); + const myITwinId = HubMock.iTwinId; + const masterIModelId = await IModelHost.hubAccess.createNewIModel({ + iTwinId: myITwinId, + iModelName: "my-branch-imodel", + version0: version0Path, + noLocks: true, + }); + const { masterDb, branchDb } = await initializeBranch( + myITwinId, + masterIModelId, + myAccessToken + ); + await arbitraryEdit(masterDb, myAccessToken, "edit master"); + await forwardSyncMasterToBranch(masterDb, branchDb, myAccessToken); + await arbitraryEdit(branchDb, myAccessToken, "edit branch"); + await reverseSyncBranchToMaster(branchDb, masterDb, myAccessToken); + masterDb.close(); + branchDb.close(); + }); +}); From 03c6e8bd1a7f41f7efe46f1ebccb6679445db701 Mon Sep 17 00:00:00 2001 From: "nick.tessier" <22119573+nick4598@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:46:58 -0500 Subject: [PATCH 2/4] test is passing now --- .../IModelBranchingOperations.test.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts index 0001046d..6c6b6922 100644 --- a/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts +++ b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts @@ -31,6 +31,7 @@ import { ExternalSourceProps, IModel, PhysicalElementProps, + QueryBinder, RepositoryLinkProps, SubCategoryAppearance, } from "@itwin/core-common"; @@ -130,13 +131,20 @@ async function forwardSyncMasterToBranch( myAccessToken: AccessToken ) { // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_forwardSync - const masterExternalSourceId = branchDb.elements.queryElementIdByCode( + const repositoryLinkId = branchDb.elements.queryElementIdByCode( RepositoryLink.createCode( masterDb, IModelDb.repositoryModelId, "example-code-value" ) ); + let masterExternalSourceId; + for await (const row of branchDb.createQueryReader( + `SELECT ECInstanceId FROM ${ExternalSource.classFullName} WHERE Repository.Id=:id`, + QueryBinder.from({ id: repositoryLinkId }) + )) { + masterExternalSourceId = row.ECInstanceId; + } const synchronizer = new IModelTransformer(masterDb, branchDb, { // read the synchronization provenance in the scope of our representation of the external source, master targetScopeElementId: masterExternalSourceId, @@ -163,13 +171,20 @@ async function reverseSyncBranchToMaster( ) { // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_reverseSync // we assume masterDb and branchDb have already been opened (see the first example) - const masterExternalSourceId = branchDb.elements.queryElementIdByCode( + const repositoryLinkId = branchDb.elements.queryElementIdByCode( RepositoryLink.createCode( masterDb, IModelDb.repositoryModelId, "example-code-value" ) ); + let masterExternalSourceId; + for await (const row of branchDb.createQueryReader( + `SELECT ECInstanceId FROM ${ExternalSource.classFullName} WHERE Repository.Id=:id`, + QueryBinder.from({ id: repositoryLinkId }) + )) { + masterExternalSourceId = row.ECInstanceId; + } const reverseSynchronizer = new IModelTransformer(branchDb, masterDb, { // read the synchronization provenance in the scope of our representation of the external source, master // "isReverseSynchronization" actually causes the provenance (and therefore the targetScopeElementId) to From 6115919b547cca885a9eb10f9d069d9586c5e6d2 Mon Sep 17 00:00:00 2001 From: "nick.tessier" <22119573+nick4598@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:41:46 -0500 Subject: [PATCH 3/4] small changes to docs --- docs/learning/transformer/branching-imodels.md | 4 ++-- docs/learning/transformer/index.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/learning/transformer/branching-imodels.md b/docs/learning/transformer/branching-imodels.md index cce06908..c8c58a55 100644 --- a/docs/learning/transformer/branching-imodels.md +++ b/docs/learning/transformer/branching-imodels.md @@ -90,7 +90,7 @@ forward synchronizations. Conflicts during a transformation are resolved in favor of the element which was modified most recently, as stored in the `LastMod` property of an element. Elements in transformations are considered in conflict when their [code](/bis/fundamentals/foundation/codes) is the same. -You can override the method [`IModelTransformer.hasElementChanged`](/reference/core-transformer/imodels/imodeltransformer/haselementchanged/) +You can override the method [`IModelTransformer.hasElementChanged`](/reference/imodel-transformer/imodels/imodeltransformer/haselementchanged/) in your transformer implementation to use more specific logic for determining if an element should be considered changed. Some other data in the iModel follows more specific rules for conflicts: @@ -124,4 +124,4 @@ Synchronization conflicts are not to be confused with concurrent edit conflicts ### Synchronization workflow examples -More in depth samples exist in the [tests](https://github.com/iTwin/itwinjs-core/blob/master/core/transformer/src/test/standalone/IModelTransformerHub.test.ts) for the `@itwin/core-transformer` package. +More in depth samples exist in the [tests](https://github.com/iTwin/imodel-transformer/blob/main/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts) for the `@itwin/imodel-transformer` package. diff --git a/docs/learning/transformer/index.md b/docs/learning/transformer/index.md index 4cb8e04d..4fe82935 100644 --- a/docs/learning/transformer/index.md +++ b/docs/learning/transformer/index.md @@ -1,6 +1,6 @@ # iModel Transformation and Data Exchange -The `@itwin/core-transformer` package provides some classes that implement [Extract, Transform, and Load](https://en.wikipedia.org/wiki/Extract,_transform,_load) (ETL) functionality: +The `@itwin/imodel-transformer` package provides some classes that implement [Extract, Transform, and Load](https://en.wikipedia.org/wiki/Extract,_transform,_load) (ETL) functionality: - [IModelExporter]($transformer) and [IModelExportHandler]($transformer) are the base classes that implement the *extract* (or *export*) part of ETL functionality. - [IModelTransformer]($transformer) is the base class that implements the *transform* part of ETL functionality. From d5c0e8b36a00ea5ee51c88b0e9443e42e4d6fe0d Mon Sep 17 00:00:00 2001 From: "nick.tessier" <22119573+nick4598@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:01:39 -0500 Subject: [PATCH 4/4] copy learning folder over to build folder (docs output) --- packages/transformer/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/transformer/package.json b/packages/transformer/package.json index ff7a5f4c..373e4ff1 100644 --- a/packages/transformer/package.json +++ b/packages/transformer/package.json @@ -13,8 +13,9 @@ "build:ci": "npm run -s build", "build:cjs": "tsc 1>&2 --outDir lib/cjs", "clean": "rimraf lib", - "docs": "npm run -s docs:extract && npm run -s docs:reference && npm run -s docs:changelog", + "docs": "npm run -s docs:extract && npm run -s docs:reference && npm run -s docs:changelog && npm run -s docs:copyLearning", "docs:changelog": "cpx ./CHANGELOG.md ../../build/docs/reference/imodel-transformer", + "docs:copyLearning": "cpx \"../../docs/learning/transformer/**/*\" ../../build/docs/learning/transformer", "# env var is workaround, need to contribute a better rush-less root-package.json detector to betools": "", "docs:reference": "cross-env RUSHSTACK_FILE_ERROR_BASE_FOLDER='../..' betools docs --includes=../../build/docs/extract --json=../../build/docs/reference/imodel-transformer/file.json --tsIndexFile=imodel-transformer.ts --onlyJson", "docs:extract": "betools extract --fileExt=ts --extractFrom=./src/test --recursive --out=../../build/docs/extract",