Skip to content

v3: fix prod worker node_modules permissions #1008

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/few-students-share.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"trigger.dev": patch
---

Fix permissions inside node_modules
2 changes: 2 additions & 0 deletions packages/cli-v3/src/Containerfile.prod
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ WORKDIR /app

# copy all the files just in case anything is needed in postinstall
COPY --chown=node:node . .

USER node
RUN npm ci --no-fund --no-audit && npm cache clean --force

# Development or production stage builds upon the base stage
Expand Down
178 changes: 177 additions & 1 deletion pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions references/v3-catalog/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@
"dev:trigger": "trigger.dev dev"
},
"dependencies": {
"@ffmpeg-installer/ffmpeg": "^1.1.0",
"@ffprobe-installer/ffprobe": "^2.1.2",
"@opentelemetry/api": "^1.8.0",
"@sindresorhus/slugify": "^2.2.1",
"@traceloop/instrumentation-openai": "^0.3.9",
"@trigger.dev/core": "workspace:^3.0.0-beta.0",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
"execa": "^8.0.1",
"msw": "^2.2.1",
"openai": "^4.28.0",
"stripe": "^12.14.0",
Expand Down
139 changes: 138 additions & 1 deletion references/v3-catalog/src/trigger/binaries.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
import { logger, task } from "@trigger.dev/sdk/v3";
import { chmod } from "node:fs/promises";
import { chmod, writeFile } from "node:fs/promises";
import { Readable } from "node:stream";
import { ReadableStream } from "stream/web";
import { basename } from "node:path";
import YTDlpWrap from "yt-dlp-wrap";
import ffmpeg from "@ffmpeg-installer/ffmpeg";

export const ytDlp = task({
id: "yt-dlp",
Expand All @@ -20,3 +24,136 @@ export const ytDlp = task({
logger.log("version", { version });
},
});

async function getFfprobe() {
const ffprobe = await import("@ffprobe-installer/ffprobe");

logger.log("ffprobeInstaller", ffprobe);

return ffprobe;
}

async function ffprobeVersion() {
const ffprobe = await getFfprobe();
const childProcess = await execute(ffprobe.path, ["-version"]);

logger.log("ffprobe -version", {
output: childProcess.stdout.split("\n")[0],
});
}

async function ffmpegVersion() {
logger.log("ffmpegInstaller", ffmpeg);

const childProcess = await execute(ffmpeg.path, ["-version"]);

logger.log("ffmpeg -version", {
output: childProcess.stdout.split("\n")[0],
});
}

export const ffprobeInstaller = task({
id: "ffprobe-installer",
run: async () => {
await ffprobeVersion();
},
});

export const ffmpegInstaller = task({
id: "ffmpeg-installer",
run: async () => {
await ffmpegVersion();
},
});

const videoUrl =
"https://upload.wikimedia.org/wikipedia/commons/0/07/Fractal-zoom-1-03-Mandelbrot_Buzzsaw.ogv";
const videoPath = "./video.ogv";

async function downloadVideo() {
logger.log("downloading video", { url: videoUrl });

const response = await fetch(videoUrl);

if (!response.body) {
throw new Error("No readable stream");
}

const readStream = Readable.fromWeb(response.body as ReadableStream);
await writeFile(videoPath, readStream);

logger.log("finished downloading", { outputPath: videoPath });
}

async function execute(file: string, args?: readonly string[]) {
const { execa } = await import("execa");

logger.log(`execute: ${basename(file)}`, { args });
const childProcess = await execa(file, args);

if (childProcess.exitCode !== 0) {
logger.error("Non-zero exit code", {
stderr: childProcess.stderr,
stdout: childProcess.stdout,
});
throw new Error("Non-zero exit code");
}

return childProcess;
}

async function probeVideo() {
const ffprobe = await getFfprobe();
const args = ["-hide_banner", "-print_format", "json", "-show_format", videoPath];

logger.log("probing video", { videoPath });
const childProcess = await execute(ffprobe.path, args);

logger.log("video info", {
output: JSON.parse(childProcess.stdout),
});
}

export const ffprobeInfo = task({
id: "ffprobe-info",
run: async () => {
await ffprobeVersion();
await downloadVideo();
await probeVideo();
},
});

async function convertVideo() {
const outputPath = "./video.webm";
logger.log("converting video", { input: videoPath, output: outputPath });

const childProcess = await execute(ffmpeg.path, [
"-hide_banner",
"-y", // overwrite output, don't prompt
"-i",
videoPath,
// seek to 25s
"-ss",
"25",
// stop after 5s
"-t",
"5",
outputPath,
]);

logger.log("video converted", {
input: videoPath,
output: outputPath,
stderr: childProcess.stderr,
stdout: childProcess.stdout,
});
}

export const ffmpegConvert = task({
id: "ffmpeg-convert",
run: async () => {
await ffmpegVersion();
await downloadVideo();
await convertVideo();
},
});