Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Python: Update the versions of the bundles #3624

Merged
merged 2 commits into from
Feb 28, 2025
Merged

Python: Update the versions of the bundles #3624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0d13e38
Python: Fix getPackageSnapshotImports() and add unit tests
hoodmane Feb 28, 2025
1c14c28
Python: Update the versions of the bundles
hoodmane Feb 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions src/workerd/api/pyodide/pyodide-test.c++
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,11 @@ kj::Array<kj::String> strArray(Params&&... params) {
return kj::arr(kj::str(params)...);
}

template <typename... Params>
kj::Array<kj::Array<kj::byte>> bytesArray(Params&&... params) {
return kj::arr(kj::heapArray<kj::byte>(kj::str(params).asBytes())...);
}

KJ_TEST("Simple pass through") {
auto imports = strArray("b", "c");
auto result = ArtifactBundler::filterPythonScriptImportsJs({}, kj::mv(imports));
Expand Down Expand Up @@ -256,5 +261,29 @@ KJ_TEST("Filters out subdir/submodule") {
ArtifactBundler::filterPythonScriptImportsJs(kj::mv(workerModules), kj::mv(imports));
KJ_REQUIRE(result.size() == 0);
}

KJ_TEST("basic test of getPackageSnapshotImports") {
auto a = pyodide::PythonModuleInfo(strArray("a.py"),
bytesArray("from js import Response\n"
"import asyncio\n"
"import numbers\n"
"def on_fetch(request):\n"
" return Response.new('Hello')\n"));
auto result = a.getPackageSnapshotImports();
KJ_REQUIRE(result.size() == 1);
KJ_REQUIRE(result[0] == "numbers");
};

KJ_TEST("basic test of getPackageSnapshotImports user module") {
auto a = pyodide::PythonModuleInfo(strArray("a.py", "numbers.py"),
bytesArray("from js import Response\n"
"import asyncio\n"
"import numbers\n"
"def on_fetch(request):\n"
" return Response.new('Hello')\n",
""));
auto result = a.getPackageSnapshotImports();
KJ_REQUIRE(result.size() == 0);
};
} // namespace
} // namespace workerd::api
30 changes: 17 additions & 13 deletions src/workerd/api/pyodide/pyodide.c++
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,24 +71,24 @@ int ReadOnlyBuffer::read(jsg::Lock& js, int offset, kj::Array<kj::byte> buf) {

kj::Array<jsg::JsRef<jsg::JsString>> PyodideMetadataReader::getNames(
jsg::Lock& js, jsg::Optional<kj::String> maybeExtFilter) {
auto builder = kj::Vector<jsg::JsRef<jsg::JsString>>(this->names.size());
auto builder = kj::Vector<jsg::JsRef<jsg::JsString>>(this->moduleInfo.names.size());
for (auto i: kj::zeroTo(builder.capacity())) {
KJ_IF_SOME(ext, maybeExtFilter) {
if (!this->names[i].endsWith(ext)) {
if (!this->moduleInfo.names[i].endsWith(ext)) {
continue;
}
}
builder.add(js, js.str(this->names[i]));
builder.add(js, js.str(this->moduleInfo.names[i]));
}
return builder.releaseAsArray();
}

kj::Array<jsg::JsRef<jsg::JsString>> PyodideMetadataReader::getWorkerFiles(
jsg::Lock& js, kj::String ext) {
auto builder = kj::Vector<jsg::JsRef<jsg::JsString>>(this->names.size());
for (auto i: kj::zeroTo(this->names.size())) {
if (this->names[i].endsWith(ext)) {
builder.add(js, js.str(this->contents[i]));
auto builder = kj::Vector<jsg::JsRef<jsg::JsString>>(this->moduleInfo.names.size());
for (auto i: kj::zeroTo(this->moduleInfo.names.size())) {
if (this->moduleInfo.names[i].endsWith(ext)) {
builder.add(js, js.str(this->moduleInfo.contents[i]));
}
}
return builder.releaseAsArray();
Expand All @@ -99,7 +99,7 @@ kj::Array<kj::String> _getWorkerFiles(
auto builder = kj::Vector<kj::String>(names.size());
for (auto i: kj::zeroTo(names.size())) {
if (names[i].endsWith(".py")) {
builder.add(kj::str(contents[i]));
builder.add(kj::str(contents[i].asChars()));
}
}
return builder.releaseAsArray();
Expand All @@ -116,13 +116,17 @@ kj::HashSet<kj::String> _getNames(kj::ArrayPtr<kj::String> names) {
return result;
}

kj::Array<kj::String> PyodideMetadataReader::getPackageSnapshotImports(jsg::Lock& js) {
kj::Array<kj::String> PythonModuleInfo::getPackageSnapshotImports() {
auto workerFiles = _getWorkerFiles(this->names, this->contents);
auto imports = ArtifactBundler::parsePythonScriptImports(kj::mv(workerFiles));
auto names = _getNames(this->names);
return ArtifactBundler::filterPythonScriptImports(kj::mv(names), kj::mv(imports));
}

kj::Array<kj::String> PyodideMetadataReader::getPackageSnapshotImports() {
return this->moduleInfo.getPackageSnapshotImports();
}

kj::Array<jsg::JsRef<jsg::JsString>> PyodideMetadataReader::getRequirements(jsg::Lock& js) {
auto builder = kj::heapArrayBuilder<jsg::JsRef<jsg::JsString>>(this->requirements.size());
for (auto i: kj::zeroTo(builder.capacity())) {
Expand All @@ -132,18 +136,18 @@ kj::Array<jsg::JsRef<jsg::JsString>> PyodideMetadataReader::getRequirements(jsg:
}

kj::Array<int> PyodideMetadataReader::getSizes(jsg::Lock& js) {
auto builder = kj::heapArrayBuilder<int>(this->names.size());
auto builder = kj::heapArrayBuilder<int>(this->moduleInfo.names.size());
for (auto i: kj::zeroTo(builder.capacity())) {
builder.add(this->contents[i].size());
builder.add(this->moduleInfo.contents[i].size());
}
return builder.finish();
}

int PyodideMetadataReader::read(jsg::Lock& js, int index, int offset, kj::Array<kj::byte> buf) {
if (index >= contents.size() || index < 0) {
if (index >= this->moduleInfo.contents.size() || index < 0) {
return 0;
}
auto& data = contents[index];
auto& data = this->moduleInfo.contents[index];
return readToTarget(data, offset, buf);
}

Expand Down
26 changes: 19 additions & 7 deletions src/workerd/api/pyodide/pyodide.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,20 @@ class ReadOnlyBuffer: public jsg::Object {
}
};

class PythonModuleInfo {
public:
PythonModuleInfo(kj::Array<kj::String> names, kj::Array<kj::Array<kj::byte>> contents)
: names(kj::mv(names)),
contents(kj::mv(contents)) {
KJ_REQUIRE(this->names.size() == this->contents.size());
}
kj::Array<kj::String> names;
kj::Array<kj::Array<kj::byte>> contents;

// Return the list of names to import into a package snapshot.
kj::Array<kj::String> getPackageSnapshotImports();
};

// A class wrapping the information stored in a WorkerBundle, in particular the Python source files
// and metadata about the worker.
//
Expand All @@ -75,8 +89,7 @@ class ReadOnlyBuffer: public jsg::Object {
class PyodideMetadataReader: public jsg::Object {
private:
kj::String mainModule;
kj::Array<kj::String> names;
kj::Array<kj::Array<kj::byte>> contents;
PythonModuleInfo moduleInfo;
kj::Array<kj::String> requirements;
kj::String pyodideVersion;
kj::String packagesVersion;
Expand All @@ -103,8 +116,7 @@ class PyodideMetadataReader: public jsg::Object {
bool usePackagesInArtifactBundler,
kj::Maybe<kj::Array<kj::byte>> memorySnapshot)
: mainModule(kj::mv(mainModule)),
names(kj::mv(names)),
contents(kj::mv(contents)),
moduleInfo(kj::mv(names), kj::mv(contents)),
requirements(kj::mv(requirements)),
pyodideVersion(kj::mv(pyodideVersion)),
packagesVersion(kj::mv(packagesVersion)),
Expand Down Expand Up @@ -148,7 +160,7 @@ class PyodideMetadataReader: public jsg::Object {
kj::Array<jsg::JsRef<jsg::JsString>> getWorkerFiles(jsg::Lock& js, kj::String ext);

// Return the list of names to import into a package snapshot.
kj::Array<kj::String> getPackageSnapshotImports(jsg::Lock& js);
kj::Array<kj::String> getPackageSnapshotImports();

kj::Array<jsg::JsRef<jsg::JsString>> getRequirements(jsg::Lock& js);

Expand Down Expand Up @@ -214,10 +226,10 @@ class PyodideMetadataReader: public jsg::Object {

void visitForMemoryInfo(jsg::MemoryTracker& tracker) const {
tracker.trackField("mainModule", mainModule);
for (const auto& name: names) {
for (const auto& name: this->moduleInfo.names) {
tracker.trackField("name", name);
}
for (const auto& content: contents) {
for (const auto& content: this->moduleInfo.contents) {
tracker.trackField("content", content);
}
for (const auto& requirement: requirements) {
Expand Down
6 changes: 3 additions & 3 deletions src/workerd/io/compatibility-date.capnp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,7 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
pythonWorkers @43 :Bool
$compatEnableFlag("python_workers")
$pythonSnapshotRelease(pyodide = "0.26.0a2", pyodideRevision = "2024-03-01",
packages = "20240829.4", backport = 17,
packages = "20240829.4", backport = 19,
baselineSnapshotHash = "d13ce2f4a0ade2e09047b469874dacf4d071ed3558fec4c26f8d0b99d95f77b5")
$impliedByAfterDate(name = "pythonWorkersDevPyodide", date = "2000-01-01");
# Enables Python Workers. Access to this flag is not restricted, instead bundles containing
Expand Down Expand Up @@ -684,7 +684,7 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
$compatEnableFlag("python_workers_20250116")
$experimental
$pythonSnapshotRelease(pyodide = "0.27.1", pyodideRevision = "2025-01-16",
packages = "20241218", backport = 6,
packages = "20241218", backport = 7,
baselineSnapshotHash = "TODO");

requestCfOverridesCacheRules @72 :Bool
Expand Down Expand Up @@ -721,7 +721,7 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
$impliedByAfterDate(name = "nodeJsCompat", date = "2025-04-01");
# Automatically populate process.env from text bindings only
# when nodejs_compat is being used.

cacheApiRequestCfOverridesCacheRules @77 :Bool
$compatEnableFlag("cache_api_request_cf_overrides_cache_rules")
$experimental
Expand Down
12 changes: 2 additions & 10 deletions src/workerd/server/tests/python/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,17 +75,9 @@ py_wd_test("seek-metadatafs")
for python_version, package_version in PYTHON_VERSION_TO_PACKAGES.items()
]

py_wd_test(
"undefined-handler",
# TODO: Requires a new bundle deploy
python_flags = ["development"],
)
py_wd_test("undefined-handler")

py_wd_test(
"vendor_dir",
# TODO: Requires a new bundle deploy
python_flags = ["development"],
)
py_wd_test("vendor_dir")

py_wd_test("dont-snapshot-pyodide")

Expand Down
Loading