Skip to content

Commit b7e3601

Browse files
ericallamericallam
committed
pythonExtension and python runtime improvements
1 parent 06f60f2 commit b7e3601

File tree

17 files changed

+542
-175
lines changed

17 files changed

+542
-175
lines changed

‎packages/build/package.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
"exports": {
2424
"./package.json": "./package.json",
2525
".": "./src/index.ts",
26+
"./internal": "./src/internal.ts",
2627
"./extensions": "./src/extensions/index.ts",
2728
"./extensions/core": "./src/extensions/core.ts",
2829
"./extensions/prisma": "./src/extensions/prisma.ts",
@@ -36,6 +37,9 @@
3637
},
3738
"typesVersions": {
3839
"*": {
40+
"internal": [
41+
"dist/commonjs/internal.d.ts"
42+
],
3943
"extensions": [
4044
"dist/commonjs/extensions/index.d.ts"
4145
],
@@ -95,6 +99,17 @@
9599
"default": "./dist/commonjs/index.js"
96100
}
97101
},
102+
"./internal": {
103+
"import": {
104+
"@triggerdotdev/source": "./src/internal.ts",
105+
"types": "./dist/esm/internal.d.ts",
106+
"default": "./dist/esm/internal.js"
107+
},
108+
"require": {
109+
"types": "./dist/commonjs/internal.d.ts",
110+
"default": "./dist/commonjs/internal.js"
111+
}
112+
},
98113
"./extensions": {
99114
"import": {
100115
"@triggerdotdev/source": "./src/extensions/index.ts",

‎packages/build/src/extensions/core/additionalFiles.ts

Lines changed: 2 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
1-
import { relative, join, posix, dirname } from "node:path";
2-
import { glob } from "tinyglobby";
3-
import { copyFile, mkdir } from "node:fs/promises";
41
import { BuildExtension } from "@trigger.dev/core/v3/build";
2+
import { addAdditionalFilesToBuild } from "../../internal/additionalFiles.js";
53

64
export type AdditionalFilesOptions = {
75
files: string[];
@@ -11,86 +9,7 @@ export function additionalFiles(options: AdditionalFilesOptions): BuildExtension
119
return {
1210
name: "additionalFiles",
1311
async onBuildComplete(context, manifest) {
14-
// Copy any static assets to the destination
15-
const staticAssets = await findStaticAssetFiles(options.files ?? [], manifest.outputPath, {
16-
cwd: context.workingDir,
17-
});
18-
19-
for (const { assets, matcher } of staticAssets) {
20-
if (assets.length === 0) {
21-
console.warn("No files found for matcher", matcher);
22-
}
23-
}
24-
25-
await copyStaticAssets(staticAssets);
12+
await addAdditionalFilesToBuild("additionalFiles", options, context, manifest);
2613
},
2714
};
2815
}
29-
30-
type MatchedStaticAssets = { source: string; destination: string }[];
31-
32-
type FoundStaticAssetFiles = Array<{
33-
matcher: string;
34-
assets: MatchedStaticAssets;
35-
}>;
36-
37-
async function findStaticAssetFiles(
38-
matchers: string[],
39-
destinationPath: string,
40-
options?: { cwd?: string; ignore?: string[] }
41-
): Promise<FoundStaticAssetFiles> {
42-
const result: FoundStaticAssetFiles = [];
43-
44-
for (const matcher of matchers) {
45-
const assets = await findStaticAssetsForMatcher(matcher, destinationPath, options);
46-
47-
result.push({ matcher, assets });
48-
}
49-
50-
return result;
51-
}
52-
53-
async function findStaticAssetsForMatcher(
54-
matcher: string,
55-
destinationPath: string,
56-
options?: { cwd?: string; ignore?: string[] }
57-
): Promise<MatchedStaticAssets> {
58-
const result: MatchedStaticAssets = [];
59-
60-
const files = await glob({
61-
patterns: [matcher],
62-
cwd: options?.cwd,
63-
ignore: options?.ignore ?? [],
64-
onlyFiles: true,
65-
absolute: true,
66-
});
67-
68-
let matches = 0;
69-
70-
for (const file of files) {
71-
matches++;
72-
73-
const pathInsideDestinationDir = relative(options?.cwd ?? process.cwd(), file)
74-
.split(posix.sep)
75-
.filter((p) => p !== "..")
76-
.join(posix.sep);
77-
78-
const relativeDestinationPath = join(destinationPath, pathInsideDestinationDir);
79-
80-
result.push({
81-
source: file,
82-
destination: relativeDestinationPath,
83-
});
84-
}
85-
86-
return result;
87-
}
88-
89-
async function copyStaticAssets(staticAssetFiles: FoundStaticAssetFiles): Promise<void> {
90-
for (const { assets } of staticAssetFiles) {
91-
for (const { source, destination } of assets) {
92-
await mkdir(dirname(destination), { recursive: true });
93-
await copyFile(source, destination);
94-
}
95-
}
96-
}

‎packages/build/src/internal.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export * from "./internal/additionalFiles.js";

‎packages/build/src/internal/additionalFiles.ts

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
import { BuildManifest } from "@trigger.dev/core/v3";
2+
import { BuildContext } from "@trigger.dev/core/v3/build";
3+
import { copyFile, mkdir } from "node:fs/promises";
4+
import { dirname, join, posix, relative } from "node:path";
5+
import { glob } from "tinyglobby";
6+
7+
export type AdditionalFilesOptions = {
8+
files: string[];
9+
};
10+
11+
export async function addAdditionalFilesToBuild(
12+
source: string,
13+
options: AdditionalFilesOptions,
14+
context: BuildContext,
15+
manifest: BuildManifest
16+
) {
17+
// Copy any static assets to the destination
18+
const staticAssets = await findStaticAssetFiles(options.files ?? [], manifest.outputPath, {
19+
cwd: context.workingDir,
20+
});
21+
22+
for (const { assets, matcher } of staticAssets) {
23+
if (assets.length === 0) {
24+
context.logger.warn(`[${source}] No files found for matcher`, matcher);
25+
} else {
26+
context.logger.debug(`[${source}] Found ${assets.length} files for matcher`, matcher);
27+
}
28+
}
29+
30+
await copyStaticAssets(staticAssets, source, context);
31+
}
32+
33+
type MatchedStaticAssets = { source: string; destination: string }[];
34+
35+
type FoundStaticAssetFiles = Array<{
36+
matcher: string;
37+
assets: MatchedStaticAssets;
38+
}>;
39+
40+
async function findStaticAssetFiles(
41+
matchers: string[],
42+
destinationPath: string,
43+
options?: { cwd?: string; ignore?: string[] }
44+
): Promise<FoundStaticAssetFiles> {
45+
const result: FoundStaticAssetFiles = [];
46+
47+
for (const matcher of matchers) {
48+
const assets = await findStaticAssetsForMatcher(matcher, destinationPath, options);
49+
50+
result.push({ matcher, assets });
51+
}
52+
53+
return result;
54+
}
55+
56+
async function findStaticAssetsForMatcher(
57+
matcher: string,
58+
destinationPath: string,
59+
options?: { cwd?: string; ignore?: string[] }
60+
): Promise<MatchedStaticAssets> {
61+
const result: MatchedStaticAssets = [];
62+
63+
const files = await glob({
64+
patterns: [matcher],
65+
cwd: options?.cwd,
66+
ignore: options?.ignore ?? [],
67+
onlyFiles: true,
68+
absolute: true,
69+
});
70+
71+
let matches = 0;
72+
73+
for (const file of files) {
74+
matches++;
75+
76+
const pathInsideDestinationDir = relative(options?.cwd ?? process.cwd(), file)
77+
.split(posix.sep)
78+
.filter((p) => p !== "..")
79+
.join(posix.sep);
80+
81+
const relativeDestinationPath = join(destinationPath, pathInsideDestinationDir);
82+
83+
result.push({
84+
source: file,
85+
destination: relativeDestinationPath,
86+
});
87+
}
88+
89+
return result;
90+
}
91+
92+
async function copyStaticAssets(
93+
staticAssetFiles: FoundStaticAssetFiles,
94+
sourceName: string,
95+
context: BuildContext
96+
): Promise<void> {
97+
for (const { assets } of staticAssetFiles) {
98+
for (const { source, destination } of assets) {
99+
await mkdir(dirname(destination), { recursive: true });
100+
101+
context.logger.debug(`[${sourceName}] Copying ${source} to ${destination}`);
102+
103+
await copyFile(source, destination);
104+
}
105+
}
106+
}

‎packages/core/src/v3/logger/index.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { NoopTaskLogger, TaskLogger } from "./taskLogger.js";
22
import { getGlobal, registerGlobal, unregisterGlobal } from "../utils/globals.js";
3-
import { Span } from "@opentelemetry/api";
3+
import { Span, SpanOptions } from "@opentelemetry/api";
44

55
const API_NAME = "logger";
66

@@ -47,8 +47,8 @@ export class LoggerAPI implements TaskLogger {
4747
this.#getTaskLogger().error(message, metadata);
4848
}
4949

50-
public trace<T>(name: string, fn: (span: Span) => Promise<T>): Promise<T> {
51-
return this.#getTaskLogger().trace(name, fn);
50+
public trace<T>(name: string, fn: (span: Span) => Promise<T>, options?: SpanOptions): Promise<T> {
51+
return this.#getTaskLogger().trace(name, fn, options);
5252
}
5353

5454
#getTaskLogger(): TaskLogger {

‎packages/python/README.md

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,24 +14,24 @@ This extension introduces the <code>pythonExtension</code> build extension, whic
1414
- <code>run</code>: Executes Python commands with proper environment setup.
1515
- <code>runInline</code>: Executes inline Python code directly from Node.
1616
- <code>runScript</code>: Executes standalone <code>.py</code> script files.
17-
- **Custom Python Path:** In development, you can configure <code>pythonBinaryPath</code> to point to a custom Python installation.
17+
- **Custom Python Path:** In development, you can configure <code>devPythonBinaryPath</code> to point to a custom Python installation.
1818

1919
## Usage
2020

2121
1. Add the extension to your <code>trigger.config.ts</code> file:
2222

2323
```typescript
2424
import { defineConfig } from "@trigger.dev/sdk/v3";
25-
import pythonExtension from "@trigger.dev/python/extension";
25+
import { pythonExtension } from "@trigger.dev/python/extension";
2626

2727
export default defineConfig({
2828
project: "<project ref>",
2929
build: {
3030
extensions: [
3131
pythonExtension({
3232
requirementsFile: "./requirements.txt", // Optional: Path to your requirements file
33-
pythonBinaryPath: path.join(rootDir, `.venv/bin/python`), // Optional: Custom Python binary path
34-
scripts: ["my_script.py"], // List of Python scripts to include
33+
devPythonBinaryPath: ".venv/bin/python", // Optional: Custom Python binary path
34+
scripts: ["src/python/**/*.py"], // Glob pattern for Python scripts
3535
}),
3636
],
3737
},
@@ -40,13 +40,34 @@ export default defineConfig({
4040

4141
2. (Optional) Create a <code>requirements.txt</code> file in your project root with the necessary Python dependencies.
4242

43+
```plaintext title="requirements.txt"
44+
pandas==1.3.3
45+
numpy==1.21.2
46+
```
47+
48+
```typescript title="trigger.config.ts"
49+
import { defineConfig } from "@trigger.dev/sdk/v3";
50+
import { pythonExtension } from "@trigger.dev/python/extension";
51+
52+
export default defineConfig({
53+
project: "<project ref>",
54+
build: {
55+
extensions: [
56+
pythonExtension({
57+
requirementsFile: "./requirements.txt",
58+
}),
59+
],
60+
},
61+
});
62+
```
63+
4364
3. Execute Python scripts within your tasks using one of the provided functions:
4465

4566
### Running a Python Script
4667

4768
```typescript
4869
import { task } from "@trigger.dev/sdk/v3";
49-
import python from "@trigger.dev/python";
70+
import { python } from "@trigger.dev/python";
5071

5172
export const myScript = task({
5273
id: "my-python-script",
@@ -61,15 +82,15 @@ export const myScript = task({
6182

6283
```typescript
6384
import { task } from "@trigger.dev/sdk/v3";
64-
import python from "@trigger.dev/python";
85+
import { python } from "@trigger.dev/python";
6586

6687
export const myTask = task({
6788
id: "to_datetime-task",
6889
run: async () => {
6990
const result = await python.runInline(`
7091
import pandas as pd
7192
72-
pandas.to_datetime("${+new Date() / 1000}")
93+
pd.to_datetime("${+new Date() / 1000}")
7394
`);
7495
return result.stdout;
7596
},
@@ -80,7 +101,7 @@ pandas.to_datetime("${+new Date() / 1000}")
80101

81102
```typescript
82103
import { task } from "@trigger.dev/sdk/v3";
83-
import python from "@trigger.dev/python";
104+
import { python } from "@trigger.dev/python";
84105

85106
export const pythonVersionTask = task({
86107
id: "python-version-task",
@@ -94,7 +115,6 @@ export const pythonVersionTask = task({
94115
## Limitations
95116

96117
- This is a **partial implementation** and does not provide full Python support as an execution runtime for tasks.
97-
- Only basic Python script execution is supported; scripts are not automatically copied to staging/production containers.
98118
- Manual intervention may be required for installing and configuring binary dependencies in development environments.
99119

100120
## Additional Information

‎packages/python/package.json

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,7 @@
4545
"check-exports": "attw --pack ."
4646
},
4747
"dependencies": {
48-
"@trigger.dev/build": "workspace:3.3.16",
4948
"@trigger.dev/core": "workspace:3.3.16",
50-
"@trigger.dev/sdk": "workspace:3.3.16",
5149
"tinyexec": "^0.3.2"
5250
},
5351
"devDependencies": {
@@ -57,7 +55,13 @@
5755
"typescript": "^5.5.4",
5856
"tsx": "4.17.0",
5957
"esbuild": "^0.23.0",
60-
"@arethetypeswrong/cli": "^0.15.4"
58+
"@arethetypeswrong/cli": "^0.15.4",
59+
"@trigger.dev/build": "workspace:3.3.16",
60+
"@trigger.dev/sdk": "workspace:3.3.16"
61+
},
62+
"peerDependencies": {
63+
"@trigger.dev/sdk": "workspace:^3.3.16",
64+
"@trigger.dev/build": "workspace:^3.3.16"
6165
},
6266
"engines": {
6367
"node": ">=18.20.0"

0 commit comments

Comments
 (0)