Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(scaffolder): add scaffolder command #277

Draft
wants to merge 8 commits into
base: main
Choose a base branch
from
Draft
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
438 changes: 279 additions & 159 deletions package-lock.json

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@
"@types/diff": "^5.2.1",
"@types/eslint__js": "^8.42.3",
"@types/express": "^4.17.17",
"@types/mustache": "^4.2.5",
"@types/node": "^18.7.13",
"@types/prompts": "2.4.9",
"@types/validate-npm-package-name": "4.0.2",
@@ -58,13 +59,17 @@
"@segment/analytics-node": "^1.0.0-beta.26",
"axios": "^1.4.0",
"axios-debug-log": "^1.0.0",
"buffer": "^6.0.3",

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where is this dependency being used in?

"chalk": "^5.2.0",
"cli-table": "^0.3.11",
"crypto-random-string": "^5.0.0",
"diff": "^5.2.0",
"inquirer": "^9.2.6",
"mustache": "^4.2.0",
"octokit": "^4.0.2",
"open": "^10.1.0",
"openid-client": "^5.6.5",
"p-limit": "^6.1.0",
"prompts": "2.4.2",
"validate-npm-package-name": "5.0.1",
"which": "^3.0.1",
13 changes: 13 additions & 0 deletions src/commands/__snapshots__/scaffold.test.ts.snap
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html

exports[`scaffold > list 1`] = `
"- name: astro
- name: laravel-11
"
`;

exports[`scaffold > start validate template is set 1`] = `""`;

exports[`scaffold > start with project id 1`] = `""`;

exports[`scaffold > start without project id 1`] = `""`;
2 changes: 2 additions & 0 deletions src/commands/index.ts
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ import * as projects from './projects.js';
import * as ipAllow from './ip_allow.js';
import * as users from './user.js';
import * as orgs from './orgs.js';
import * as scaffold from './scaffold.js';
import * as branches from './branches.js';
import * as databases from './databases.js';
import * as roles from './roles.js';
@@ -15,6 +16,7 @@ export default [
auth,
users,
orgs,
scaffold,
projects,
ipAllow,
branches,
4 changes: 3 additions & 1 deletion src/commands/projects.ts
Original file line number Diff line number Diff line change
@@ -208,7 +208,7 @@ const list = async (props: CommonProps & { orgId?: string }) => {
out.end();
};

const create = async (
export const create = async (
props: CommonProps & {
name?: string;
regionId?: string;
@@ -266,6 +266,8 @@ const create = async (
const psqlArgs = props['--'];
await psql(connection_uri, psqlArgs);
}

return data;
};

const deleteProject = async (props: CommonProps & IdOrNameProps) => {
29 changes: 29 additions & 0 deletions src/commands/scaffold.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import { describe } from 'vitest';

import { test } from '../test_utils/fixtures';

describe('scaffold', () => {
test('list', async ({ testCliCommand }) => {
await testCliCommand(['scaffold', 'list']);
});

test('start validate template is set', async ({ testCliCommand }) => {
await testCliCommand(['scaffold', 'start'], {
code: 1,
});
});

test('start with project id', async ({ testCliCommand }) => {
await testCliCommand([
'scaffold',
'start',
'test-template',
'--project-id',
'test',
]);
});

test('start without project id', async ({ testCliCommand }) => {
await testCliCommand(['scaffold', 'start', 'test-template']);
});
});
257 changes: 257 additions & 0 deletions src/commands/scaffold.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,257 @@
import yargs from 'yargs';

import { CommonProps, IdOrNameProps } from '../types.js';
import { writer } from '../writer.js';
import { log } from '../log.js';
import path from 'path';
import fs from 'fs';

import Mustache from 'mustache';

import { projectCreateRequest } from '../parameters.gen.js';

import { create as createProject } from '../commands/projects.js';

import {
downloadFolderFromTree,
getContent,
getFileContent,
} from '../utils/github.js';

const TEMPLATE_LIST_FIELDS = ['name'] as const;

const REPOSITORY_OWNER = 'neon-scaffolder';
const REPOSITORY = 'templates';

const GENERATED_FOLDER_PREFIX = 'neon-';

// TODO: Maybe move to constants file?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's remove the TODO, colocation is more than fine here 👍

const REGIONS = [
'aws-us-west-2',
'aws-ap-southeast-1',
'aws-eu-central-1',
'aws-us-east-2',
'aws-us-east-1',
];

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also include Azure regions by now? cc @davidgomes


export const command = 'scaffold';
export const describe = 'Create new project from selected template';
export const aliases = ['scaffold'];
export const builder = (argv: yargs.Argv) => {
return argv
.usage('$0 scaffold [options]')
.command(
'list',
'List available templates',
(yargs) => yargs,
async (args) => {
// @ts-expect-error: TODO - Assert `args` is `CommonProps`

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's make use of the yargs.Argv generic and remove this comment:

e.g.

export const builder = (argv: yargs.Argv<CommonProps>) => {

await list(args);
},
)
.command(
'start <id>',
'Create new project from selected template',
(yargs) =>
yargs.options({
'project-id': {
describe:
'ID of existing project. If not set, new project will be created',
type: 'string',
},
name: {
describe: projectCreateRequest['project.name'].description,
type: 'string',
},
'output-dir': {
describe: 'Output directory',
type: 'string',
},
'region-id': {
describe: `The region ID. Possible values: ${REGIONS.join(', ')}`,
type: 'string',
},
'org-id': {
describe: "The project's organization ID",
type: 'string',
},
psql: {
type: 'boolean',
describe: 'Connect to a new project via psql',
default: false,
},
database: {
describe:
projectCreateRequest['project.branch.database_name'].description,
type: 'string',
},
role: {
describe:
projectCreateRequest['project.branch.role_name'].description,
type: 'string',
},
'set-context': {
type: 'boolean',
describe: 'Set the current context to the new project',
default: false,
},
cu: {
describe:
'The number of Compute Units. Could be a fixed size (e.g. "2") or a range delimited by a dash (e.g. "0.5-3").',
type: 'string',
},
}),
async (args) => {
// @ts-expect-error: TODO - Assert `args` is `CommonProps`
await start(args);
},
);
};
export const handler = (args: yargs.Argv) => {
return args;
};

async function getTemplateList() {
const content = await getContent(REPOSITORY_OWNER, REPOSITORY);

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we access that this exists and that it is an array before iterating?


return content
.filter((el: any) => el.type === 'dir')
.map((el: any) => ({ name: el.name }));

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

note: It's a bit of a pity that Octokit types don't work out of the box, these any are a bit annoying, but I do understand the reasoning

}

const list = async (props: CommonProps) => {
const out = writer(props);

out.write(await getTemplateList(), {
fields: TEMPLATE_LIST_FIELDS,
title: 'Templates',
});
out.end();
};

const start = async (
props: CommonProps &
IdOrNameProps & {
id: string;
projectId?: string;
outputDir?: string;
name?: string;
regionId?: string;
cu?: string;
orgId?: string;
database?: string;
role?: string;
psql: boolean;
setContext: boolean;
'--'?: string[];
},
) => {
const availableTemplates = (await getTemplateList()).map(
(el: any) => el.name,
);
if (!availableTemplates.includes(props.id)) {
log.error(
'Template not found. Please make sure the template exists and is public.',

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: let's add the templateID to this error message, should be easier to follow up from a user perspective

);
return;
}
let projectData: any;
if (!props.projectId) {
projectData = await createProject(props);
} else {
projectData = (await props.apiClient.getProject(props.projectId)).data;
const branches: any = (
await props.apiClient.listProjectBranches(props.projectId)
).data.branches;
const roles: any = (
await props.apiClient.listProjectBranchRoles(
props.projectId,
branches[0].id,
)
).data.roles;

const connectionString: any = (
await props.apiClient.getConnectionUri({
projectId: props.projectId,
database_name: branches[0].name,
role_name: roles[0].name,
})
).data.uri;
const connectionUrl = new URL(connectionString);

projectData.connection_uris = [
{
connection_uri: connectionString,
connection_parameters: {
database: projectData.project.name,
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The project name is not the database name, project names could have spaces and emoji etc. I guess you could try to parse out the path from the URL but there might be a better way.

role: roles[0].name,
password: connectionUrl.password,
host: connectionUrl.host,
},
},
];
}

let config = null;
try {
config = await (
await getFileContent(
REPOSITORY_OWNER,
REPOSITORY,
props.id + '/config.neon.json',
)
).json();
} catch (e) {
log.error(
"Couldn't fetch template config file. Please make sure the template exists and is public.",
);
log.error(e);
return;
}

const dir = path.join(
process.cwd(),
props.outputDir
? props.outputDir
: GENERATED_FOLDER_PREFIX + (projectData.project.name as string),
);
await downloadFolderFromTree(
REPOSITORY_OWNER,
REPOSITORY,
'main',
props.id,
dir,
);

for (const [key, value] of Object.entries(config.copy_files)) {
copyFiles(dir, key, value);
}

for (const file of config.templated_files) {
const content = fs.readFileSync(path.join(dir, file), {
encoding: 'utf-8',
});

const output = Mustache.render(content, projectData);
fs.writeFileSync(path.join(dir, file), output);
}

const out = writer(props);

out.write(
[{ name: projectData.project.name, template: props.id, path: dir }],
{
fields: ['name', 'template', 'path'],
title: 'Created projects',
},
);
out.end();
};

function copyFiles(prefix: string, sourceFile: string, targetFiles: any) {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we do targetFiles: Array<string> instead?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Couldn't change it to Array, as linter was throwing error

Array type using 'Array<string>' is forbidden. Use 'string[]' instead  @typescript-eslint/array-type

but I updated it to string[]

for (const targetFile of targetFiles) {
const sourcePath = path.join(prefix, sourceFile);
const targetPath = path.join(prefix, targetFile);
fs.copyFileSync(sourcePath, targetPath);
}
}
1 change: 1 addition & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
@@ -146,6 +146,7 @@ builder = builder
default: true,
})
.middleware(analyticsMiddleware, true)
.version(pkg.version)
.group('version', 'Global options:')
.alias('version', 'v')
.completion()
127 changes: 127 additions & 0 deletions src/utils/github.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import { Octokit } from 'octokit';
import axios from 'axios';
import pLimit from 'p-limit';

import { log } from '../log.js';

import path from 'path';
import fs from 'fs';

const MULTITHREADING_LIMIT = 10;

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: can we rename this to CONCURRENT_OPERATIONS_LIMIT? The current name is clear, but it's not threads that we are referring here


export async function getContent(owner: string, repository: string) {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: can we use an object to pass the params instead of 2 string args? It just makes it a bit less error-prone:

e.g.

export async function getContent({owner, repository}: {owner: string, repository: string}) {
...
}

const octokit = new Octokit({});
return (
await octokit.rest.repos.getContent({
owner: owner,
repo: repository,
})
).data;
}

export async function getFileContent(
owner: string,
repository: string,
path: string,
) {
const url =
'https://raw.githubusercontent.com/' +
owner +
'/' +
repository +
'/main/' +
path;

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we extract this into a util function that builds the URL? Since it's used in multiple place

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Extracted into own util function 👍

return await fetch(url, { method: 'Get' });
}

export async function getBranchSHA(
owner: string,
repo: string,
branch: string,
) {
const branchUrl = `https://api.github.com/repos/${owner}/${repo}/branches/${branch}`;
const response = await axios.get(branchUrl, {
headers: { Accept: 'application/vnd.github.v3+json' },
});
return response.data.commit.sha;
}

export async function getRepoTree(owner: string, repo: string, sha: string) {
const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${sha}?recursive=1`;
const response = await axios.get(treeUrl, {
headers: { Accept: 'application/vnd.github.v3+json' },
});
return response.data.tree;
}

export async function downloadFolderFromTree(
owner: string,
repo: string,
branch: string,
folderPath: string,
destination: string,
) {
try {
const sha = await getBranchSHA(owner, repo, branch);

const tree = await getRepoTree(owner, repo, sha);

const folderTree = tree.filter(
(item: any) => item.path.startsWith(folderPath) && item.type === 'blob',
);

for (const file of folderTree) {
const savePath = path.join(
destination,
file.path.replace(folderPath, ''),
);
const fileDir = path.dirname(savePath);
if (!fs.existsSync(fileDir)) {
fs.mkdirSync(fileDir, { recursive: true });
}
}

const limit = pLimit(MULTITHREADING_LIMIT);

const downloadPromises = folderTree.map((file: any) => {
const filePath = file.path;
const savePath = path.join(destination, filePath.replace(folderPath, ''));
const fileUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${filePath}`;

return limit(() =>
downloadFile(fileUrl, savePath)
.then(() => {
log.debug(`Downloaded ${filePath}`);
})
.catch((err: unknown) => {
let errorMessage = `Error downloading ${filePath}`;
if (err instanceof Error) {
errorMessage += ': ' + err.message;
}
log.error(errorMessage);
}),
);
});

await Promise.all(downloadPromises);

log.info('All files downloaded successfully.');
} catch (error: any) {
log.error('Error downloading folder:', error.message);
}
}

export async function downloadFile(url: string, savePath: string) {
const response = await axios({
url,
method: 'GET',
responseType: 'stream',
});
const writer = fs.createWriteStream(savePath);
response.data.pipe(writer);

return new Promise((resolve, reject) => {
writer.on('finish', resolve);
writer.on('error', reject);
});
}