-
Notifications
You must be signed in to change notification settings - Fork 20
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(scaffolder): add scaffolder command #277
base: main
Are you sure you want to change the base?
Changes from 2 commits
44e161c
26cccea
51fe65f
5832af9
9b3c043
d67f019
a1ab1cb
1ce5f06
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html | ||
|
||
exports[`scaffold > list 1`] = ` | ||
"- name: astro | ||
- name: laravel-11 | ||
" | ||
`; | ||
|
||
exports[`scaffold > start validate template is set 1`] = `""`; | ||
|
||
exports[`scaffold > start with project id 1`] = `""`; | ||
|
||
exports[`scaffold > start without project id 1`] = `""`; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
import { describe } from 'vitest'; | ||
|
||
import { test } from '../test_utils/fixtures'; | ||
|
||
describe('scaffold', () => { | ||
test('list', async ({ testCliCommand }) => { | ||
await testCliCommand(['scaffold', 'list']); | ||
}); | ||
|
||
test('start validate template is set', async ({ testCliCommand }) => { | ||
await testCliCommand(['scaffold', 'start'], { | ||
code: 1, | ||
}); | ||
}); | ||
|
||
test('start with project id', async ({ testCliCommand }) => { | ||
await testCliCommand([ | ||
'scaffold', | ||
'start', | ||
'test-template', | ||
'--project-id', | ||
'test', | ||
]); | ||
}); | ||
|
||
test('start without project id', async ({ testCliCommand }) => { | ||
await testCliCommand(['scaffold', 'start', 'test-template']); | ||
}); | ||
}); |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,257 @@ | ||
import yargs from 'yargs'; | ||
|
||
import { CommonProps, IdOrNameProps } from '../types.js'; | ||
import { writer } from '../writer.js'; | ||
import { log } from '../log.js'; | ||
import path from 'path'; | ||
import fs from 'fs'; | ||
|
||
import Mustache from 'mustache'; | ||
|
||
import { projectCreateRequest } from '../parameters.gen.js'; | ||
|
||
import { create as createProject } from '../commands/projects.js'; | ||
|
||
import { | ||
downloadFolderFromTree, | ||
getContent, | ||
getFileContent, | ||
} from '../utils/github.js'; | ||
|
||
const TEMPLATE_LIST_FIELDS = ['name'] as const; | ||
|
||
const REPOSITORY_OWNER = 'neon-scaffolder'; | ||
const REPOSITORY = 'templates'; | ||
|
||
const GENERATED_FOLDER_PREFIX = 'neon-'; | ||
|
||
// TODO: Maybe move to constants file? | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's remove the |
||
const REGIONS = [ | ||
'aws-us-west-2', | ||
'aws-ap-southeast-1', | ||
'aws-eu-central-1', | ||
'aws-us-east-2', | ||
'aws-us-east-1', | ||
]; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we also include Azure regions by now? cc @davidgomes |
||
|
||
export const command = 'scaffold'; | ||
export const describe = 'Create new project from selected template'; | ||
export const aliases = ['scaffold']; | ||
export const builder = (argv: yargs.Argv) => { | ||
return argv | ||
.usage('$0 scaffold [options]') | ||
.command( | ||
'list', | ||
'List available templates', | ||
(yargs) => yargs, | ||
async (args) => { | ||
// @ts-expect-error: TODO - Assert `args` is `CommonProps` | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's make use of the e.g. export const builder = (argv: yargs.Argv<CommonProps>) => { |
||
await list(args); | ||
}, | ||
) | ||
.command( | ||
'start <id>', | ||
'Create new project from selected template', | ||
(yargs) => | ||
yargs.options({ | ||
'project-id': { | ||
describe: | ||
'ID of existing project. If not set, new project will be created', | ||
type: 'string', | ||
}, | ||
name: { | ||
describe: projectCreateRequest['project.name'].description, | ||
type: 'string', | ||
}, | ||
'output-dir': { | ||
describe: 'Output directory', | ||
type: 'string', | ||
}, | ||
'region-id': { | ||
describe: `The region ID. Possible values: ${REGIONS.join(', ')}`, | ||
type: 'string', | ||
}, | ||
'org-id': { | ||
describe: "The project's organization ID", | ||
type: 'string', | ||
}, | ||
psql: { | ||
type: 'boolean', | ||
describe: 'Connect to a new project via psql', | ||
default: false, | ||
}, | ||
database: { | ||
describe: | ||
projectCreateRequest['project.branch.database_name'].description, | ||
type: 'string', | ||
}, | ||
role: { | ||
describe: | ||
projectCreateRequest['project.branch.role_name'].description, | ||
type: 'string', | ||
}, | ||
'set-context': { | ||
type: 'boolean', | ||
describe: 'Set the current context to the new project', | ||
default: false, | ||
}, | ||
cu: { | ||
describe: | ||
'The number of Compute Units. Could be a fixed size (e.g. "2") or a range delimited by a dash (e.g. "0.5-3").', | ||
type: 'string', | ||
}, | ||
}), | ||
async (args) => { | ||
// @ts-expect-error: TODO - Assert `args` is `CommonProps` | ||
await start(args); | ||
}, | ||
); | ||
}; | ||
export const handler = (args: yargs.Argv) => { | ||
return args; | ||
}; | ||
|
||
async function getTemplateList() { | ||
const content = await getContent(REPOSITORY_OWNER, REPOSITORY); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we access that this exists and that it is an array before iterating? |
||
|
||
return content | ||
.filter((el: any) => el.type === 'dir') | ||
.map((el: any) => ({ name: el.name })); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. note: It's a bit of a pity that |
||
} | ||
|
||
const list = async (props: CommonProps) => { | ||
const out = writer(props); | ||
|
||
out.write(await getTemplateList(), { | ||
fields: TEMPLATE_LIST_FIELDS, | ||
title: 'Templates', | ||
}); | ||
out.end(); | ||
}; | ||
|
||
const start = async ( | ||
props: CommonProps & | ||
IdOrNameProps & { | ||
id: string; | ||
projectId?: string; | ||
outputDir?: string; | ||
name?: string; | ||
regionId?: string; | ||
cu?: string; | ||
orgId?: string; | ||
database?: string; | ||
role?: string; | ||
psql: boolean; | ||
setContext: boolean; | ||
'--'?: string[]; | ||
}, | ||
) => { | ||
const availableTemplates = (await getTemplateList()).map( | ||
(el: any) => el.name, | ||
); | ||
if (!availableTemplates.includes(props.id)) { | ||
log.error( | ||
'Template not found. Please make sure the template exists and is public.', | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: let's add the |
||
); | ||
return; | ||
} | ||
let projectData: any; | ||
if (!props.projectId) { | ||
projectData = await createProject(props); | ||
} else { | ||
projectData = (await props.apiClient.getProject(props.projectId)).data; | ||
const branches: any = ( | ||
await props.apiClient.listProjectBranches(props.projectId) | ||
).data.branches; | ||
const roles: any = ( | ||
await props.apiClient.listProjectBranchRoles( | ||
props.projectId, | ||
branches[0].id, | ||
) | ||
).data.roles; | ||
|
||
const connectionString: any = ( | ||
await props.apiClient.getConnectionUri({ | ||
projectId: props.projectId, | ||
database_name: branches[0].name, | ||
role_name: roles[0].name, | ||
}) | ||
).data.uri; | ||
const connectionUrl = new URL(connectionString); | ||
|
||
projectData.connection_uris = [ | ||
{ | ||
connection_uri: connectionString, | ||
connection_parameters: { | ||
database: projectData.project.name, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The project name is not the database name, project names could have spaces and emoji etc. I guess you could try to parse out the path from the URL but there might be a better way. |
||
role: roles[0].name, | ||
password: connectionUrl.password, | ||
host: connectionUrl.host, | ||
}, | ||
}, | ||
]; | ||
} | ||
|
||
let config = null; | ||
try { | ||
config = await ( | ||
await getFileContent( | ||
REPOSITORY_OWNER, | ||
REPOSITORY, | ||
props.id + '/config.neon.json', | ||
) | ||
).json(); | ||
} catch (e) { | ||
log.error( | ||
"Couldn't fetch template config file. Please make sure the template exists and is public.", | ||
); | ||
log.error(e); | ||
return; | ||
} | ||
|
||
const dir = path.join( | ||
process.cwd(), | ||
props.outputDir | ||
? props.outputDir | ||
: GENERATED_FOLDER_PREFIX + (projectData.project.name as string), | ||
); | ||
await downloadFolderFromTree( | ||
REPOSITORY_OWNER, | ||
REPOSITORY, | ||
'main', | ||
props.id, | ||
dir, | ||
); | ||
|
||
for (const [key, value] of Object.entries(config.copy_files)) { | ||
copyFiles(dir, key, value); | ||
} | ||
|
||
for (const file of config.templated_files) { | ||
const content = fs.readFileSync(path.join(dir, file), { | ||
encoding: 'utf-8', | ||
}); | ||
|
||
const output = Mustache.render(content, projectData); | ||
fs.writeFileSync(path.join(dir, file), output); | ||
} | ||
|
||
const out = writer(props); | ||
|
||
out.write( | ||
[{ name: projectData.project.name, template: props.id, path: dir }], | ||
{ | ||
fields: ['name', 'template', 'path'], | ||
title: 'Created projects', | ||
}, | ||
); | ||
out.end(); | ||
}; | ||
|
||
function copyFiles(prefix: string, sourceFile: string, targetFiles: any) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we do There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Couldn't change it to Array, as linter was throwing error
but I updated it to |
||
for (const targetFile of targetFiles) { | ||
const sourcePath = path.join(prefix, sourceFile); | ||
const targetPath = path.join(prefix, targetFile); | ||
fs.copyFileSync(sourcePath, targetPath); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,127 @@ | ||
import { Octokit } from 'octokit'; | ||
import axios from 'axios'; | ||
import pLimit from 'p-limit'; | ||
|
||
import { log } from '../log.js'; | ||
|
||
import path from 'path'; | ||
import fs from 'fs'; | ||
|
||
const MULTITHREADING_LIMIT = 10; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: can we rename this to |
||
|
||
export async function getContent(owner: string, repository: string) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: can we use an object to pass the params instead of 2 e.g. export async function getContent({owner, repository}: {owner: string, repository: string}) {
...
} |
||
const octokit = new Octokit({}); | ||
return ( | ||
await octokit.rest.repos.getContent({ | ||
owner: owner, | ||
repo: repository, | ||
}) | ||
).data; | ||
} | ||
|
||
export async function getFileContent( | ||
owner: string, | ||
repository: string, | ||
path: string, | ||
) { | ||
const url = | ||
'https://raw.githubusercontent.com/' + | ||
owner + | ||
'/' + | ||
repository + | ||
'/main/' + | ||
path; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we extract this into a util function that builds the URL? Since it's used in multiple place There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Extracted into own util function 👍 |
||
return await fetch(url, { method: 'Get' }); | ||
} | ||
|
||
export async function getBranchSHA( | ||
owner: string, | ||
repo: string, | ||
branch: string, | ||
) { | ||
const branchUrl = `https://api.github.com/repos/${owner}/${repo}/branches/${branch}`; | ||
const response = await axios.get(branchUrl, { | ||
headers: { Accept: 'application/vnd.github.v3+json' }, | ||
}); | ||
return response.data.commit.sha; | ||
} | ||
|
||
export async function getRepoTree(owner: string, repo: string, sha: string) { | ||
const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${sha}?recursive=1`; | ||
const response = await axios.get(treeUrl, { | ||
headers: { Accept: 'application/vnd.github.v3+json' }, | ||
}); | ||
return response.data.tree; | ||
} | ||
|
||
export async function downloadFolderFromTree( | ||
owner: string, | ||
repo: string, | ||
branch: string, | ||
folderPath: string, | ||
destination: string, | ||
) { | ||
try { | ||
const sha = await getBranchSHA(owner, repo, branch); | ||
|
||
const tree = await getRepoTree(owner, repo, sha); | ||
|
||
const folderTree = tree.filter( | ||
(item: any) => item.path.startsWith(folderPath) && item.type === 'blob', | ||
); | ||
|
||
for (const file of folderTree) { | ||
const savePath = path.join( | ||
destination, | ||
file.path.replace(folderPath, ''), | ||
); | ||
const fileDir = path.dirname(savePath); | ||
if (!fs.existsSync(fileDir)) { | ||
fs.mkdirSync(fileDir, { recursive: true }); | ||
} | ||
} | ||
|
||
const limit = pLimit(MULTITHREADING_LIMIT); | ||
|
||
const downloadPromises = folderTree.map((file: any) => { | ||
const filePath = file.path; | ||
const savePath = path.join(destination, filePath.replace(folderPath, '')); | ||
const fileUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${filePath}`; | ||
|
||
return limit(() => | ||
downloadFile(fileUrl, savePath) | ||
.then(() => { | ||
log.debug(`Downloaded ${filePath}`); | ||
}) | ||
.catch((err: unknown) => { | ||
let errorMessage = `Error downloading ${filePath}`; | ||
if (err instanceof Error) { | ||
errorMessage += ': ' + err.message; | ||
} | ||
log.error(errorMessage); | ||
}), | ||
); | ||
}); | ||
|
||
await Promise.all(downloadPromises); | ||
|
||
log.info('All files downloaded successfully.'); | ||
} catch (error: any) { | ||
log.error('Error downloading folder:', error.message); | ||
} | ||
} | ||
|
||
export async function downloadFile(url: string, savePath: string) { | ||
const response = await axios({ | ||
url, | ||
method: 'GET', | ||
responseType: 'stream', | ||
}); | ||
const writer = fs.createWriteStream(savePath); | ||
response.data.pipe(writer); | ||
|
||
return new Promise((resolve, reject) => { | ||
writer.on('finish', resolve); | ||
writer.on('error', reject); | ||
}); | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Where is this dependency being used in?