From 1c014e8e43a7f353b8f5f06d05c281c1cf43a79c Mon Sep 17 00:00:00 2001 From: Alex Patterson Date: Sat, 10 Oct 2026 13:51:01 +0100 Subject: [PATCH] feat: decouple youtube analytics, add syndication schema refs, and wire devto/hashnode keys (#706) --- .github/workflows/backfill-content.yml | 40 ++ .github/workflows/deploy.yml | 4 + apps/sanity/extract.json | 398 ++++++++++++------ .../functions/sync-youtube-analytics/index.ts | 198 +++++++++ .../sync-youtube-transcript/index.ts | 101 +++-- .../functions/syndicate-content/index.ts | 43 +- apps/sanity/schemas/partials/content.ts | 9 + sanity.blueprint.ts | 26 ++ scripts/backfill-content-from-transcripts.mjs | 182 ++++++++ 9 files changed, 828 insertions(+), 173 deletions(-) create mode 100644 .github/workflows/backfill-content.yml create mode 100644 apps/sanity/functions/sync-youtube-analytics/index.ts create mode 100644 scripts/backfill-content-from-transcripts.mjs diff --git a/.github/workflows/backfill-content.yml b/.github/workflows/backfill-content.yml new file mode 100644 index 000000000..618ff0598 --- /dev/null +++ b/.github/workflows/backfill-content.yml @@ -0,0 +1,40 @@ +name: Backfill Content from Transcripts + +on: + workflow_dispatch: + inputs: + dataset: + description: "Target Sanity Dataset" + required: true + default: "production" + type: choice + options: + - production + - dev + +jobs: + backfill: + runs-on: ubuntu-latest + environment: production + + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Execute Content Backfill Script + run: | + node scripts/backfill-content-from-transcripts.mjs + env: + SANITY_AUTH_TOKEN: ${{ secrets.SANITY_AUTH_TOKEN }} + SANITY_API_WRITE_TOKEN: ${{ secrets.SANITY_API_WRITE_TOKEN }} + SANITY_STUDIO_PROJECT_ID: ${{ vars.SANITY_STUDIO_PROJECT_ID || 'hfh83o0w' }} + SANITY_STUDIO_DATASET: ${{ inputs.dataset || 'production' }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d232721d0..72d68be5c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -69,6 +69,8 @@ jobs: SANITY_STUDIO_PROJECT_ID: ${{ vars.SANITY_STUDIO_PROJECT_ID }} SANITY_STUDIO_DATASET: ${{ vars.SANITY_STUDIO_DATASET }} YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} + PRIVATE_DEVTO: ${{ secrets.PRIVATE_DEVTO }} + PRIVATE_HASHNODE: ${{ secrets.PRIVATE_HASHNODE }} deploy-production: if: github.ref == 'refs/heads/main' @@ -128,3 +130,5 @@ jobs: SANITY_STUDIO_PROJECT_ID: ${{ vars.SANITY_STUDIO_PROJECT_ID }} SANITY_STUDIO_DATASET: ${{ vars.SANITY_STUDIO_DATASET }} YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} + PRIVATE_DEVTO: ${{ secrets.PRIVATE_DEVTO }} + PRIVATE_HASHNODE: ${{ secrets.PRIVATE_HASHNODE }} diff --git a/apps/sanity/extract.json b/apps/sanity/extract.json index 197953c4f..001936da9 100644 --- a/apps/sanity/extract.json +++ b/apps/sanity/extract.json @@ -621,7 +621,7 @@ } ] }, - "optional": true + "optional": false }, "platform": { "type": "objectAttribute", @@ -638,7 +638,7 @@ } ] }, - "optional": true + "optional": false }, "status": { "type": "objectAttribute", @@ -772,7 +772,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -780,7 +780,7 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "youtube": { "type": "objectAttribute", @@ -900,28 +900,28 @@ "value": { "type": "number" }, - "optional": true + "optional": false }, "bottom": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false }, "left": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false }, "right": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false } } } @@ -944,28 +944,28 @@ "value": { "type": "number" }, - "optional": true + "optional": false }, "y": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false }, "height": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false }, "width": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false } } } @@ -988,7 +988,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "source": { "type": "objectAttribute", @@ -1040,7 +1040,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -1048,7 +1048,7 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "description": { "type": "objectAttribute", @@ -1106,7 +1106,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "contactName": { "type": "objectAttribute", @@ -1120,7 +1120,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "website": { "type": "objectAttribute", @@ -1267,7 +1267,7 @@ } ] }, - "optional": true + "optional": false }, "contentType": { "type": "objectAttribute", @@ -1288,14 +1288,14 @@ } ] }, - "optional": true + "optional": false }, "youtubeId": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "youtubeShortId": { "type": "objectAttribute", @@ -1381,7 +1381,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "image": { "type": "objectAttribute", @@ -1428,7 +1428,7 @@ } } }, - "optional": true + "optional": false }, "source": { "type": "objectAttribute", @@ -1465,7 +1465,7 @@ } ] }, - "optional": true + "optional": false }, "attribution": { "type": "objectAttribute", @@ -1663,7 +1663,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -1671,7 +1671,7 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "contentIdea": { "type": "objectAttribute", @@ -1691,7 +1691,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "scenes": { "type": "objectAttribute", @@ -1712,7 +1712,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "visualDescription": { "type": "objectAttribute", @@ -1988,7 +1988,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false } } }, @@ -2092,7 +2092,7 @@ } ] }, - "optional": true + "optional": false }, "qualityScore": { "type": "objectAttribute", @@ -3023,7 +3023,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "contactName": { "type": "objectAttribute", @@ -3037,7 +3037,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "source": { "type": "objectAttribute", @@ -3054,7 +3054,7 @@ } ] }, - "optional": true + "optional": false }, "status": { "type": "objectAttribute", @@ -3087,7 +3087,7 @@ } ] }, - "optional": true + "optional": false }, "intent": { "type": "objectAttribute", @@ -3196,7 +3196,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "summary": { "type": "objectAttribute", @@ -3220,7 +3220,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "status": { "type": "objectAttribute", @@ -3241,7 +3241,7 @@ } ] }, - "optional": true + "optional": false }, "reasonRejected": { "type": "objectAttribute", @@ -3292,14 +3292,14 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "email": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "companyName": { "type": "objectAttribute", @@ -3316,7 +3316,7 @@ "type": "string" } }, - "optional": true + "optional": false }, "message": { "type": "objectAttribute", @@ -3367,14 +3367,14 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "documentId": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "expiresAt": { "type": "objectAttribute", @@ -3456,21 +3456,21 @@ "type": "inline", "name": "coverImage" }, - "optional": true + "optional": false }, "date": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "title": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -3478,14 +3478,14 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "excerpt": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "featured": { "type": "objectAttribute", @@ -4317,7 +4317,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false } } }, @@ -4362,21 +4362,21 @@ "type": "inline", "name": "coverImage" }, - "optional": true + "optional": false }, "date": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "title": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -4384,14 +4384,14 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "excerpt": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "featured": { "type": "objectAttribute", @@ -5287,6 +5287,36 @@ "dereferencesTo": "author" } }, + { + "type": "type", + "name": "syndication.reference", + "value": { + "type": "object", + "attributes": { + "_ref": { + "type": "objectAttribute", + "value": { + "type": "string" + } + }, + "_type": { + "type": "objectAttribute", + "value": { + "type": "string", + "value": "reference" + } + }, + "_weak": { + "type": "objectAttribute", + "value": { + "type": "boolean" + }, + "optional": true + } + }, + "dereferencesTo": "syndication" + } + }, { "name": "post", "type": "document", @@ -5328,21 +5358,21 @@ "type": "inline", "name": "coverImage" }, - "optional": true + "optional": false }, "date": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "title": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -5350,14 +5380,14 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "excerpt": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "featured": { "type": "objectAttribute", @@ -6181,7 +6211,7 @@ } } }, - "optional": true + "optional": false }, "sponsor": { "type": "objectAttribute", @@ -6219,6 +6249,28 @@ }, "optional": true }, + "syndications": { + "type": "objectAttribute", + "value": { + "type": "array", + "of": { + "type": "object", + "attributes": { + "_key": { + "type": "objectAttribute", + "value": { + "type": "string" + } + } + }, + "rest": { + "type": "inline", + "name": "syndication.reference" + } + } + }, + "optional": true + }, "statistics": { "type": "objectAttribute", "value": { @@ -6458,21 +6510,21 @@ "type": "inline", "name": "coverImage" }, - "optional": true + "optional": false }, "date": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "title": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -6480,14 +6532,14 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "excerpt": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "featured": { "type": "objectAttribute", @@ -7311,7 +7363,7 @@ } } }, - "optional": true + "optional": false }, "sponsor": { "type": "objectAttribute", @@ -7349,6 +7401,28 @@ }, "optional": true }, + "syndications": { + "type": "objectAttribute", + "value": { + "type": "array", + "of": { + "type": "object", + "attributes": { + "_key": { + "type": "objectAttribute", + "value": { + "type": "string" + } + } + }, + "rest": { + "type": "inline", + "name": "syndication.reference" + } + } + }, + "optional": true + }, "statistics": { "type": "objectAttribute", "value": { @@ -7409,28 +7483,28 @@ "type": "inline", "name": "podcastType.reference" }, - "optional": true + "optional": false }, "season": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false }, "episode": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false }, "recordingDate": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "guest": { "type": "objectAttribute", @@ -7452,7 +7526,7 @@ } } }, - "optional": true + "optional": false }, "pick": { "type": "objectAttribute", @@ -7476,14 +7550,14 @@ } ] }, - "optional": true + "optional": false }, "name": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "site": { "type": "objectAttribute", @@ -7582,21 +7656,21 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "timestamp": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "seconds": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false } }, "rest": { @@ -7781,7 +7855,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -7789,7 +7863,7 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "description": { "type": "objectAttribute", @@ -8013,21 +8087,21 @@ "type": "inline", "name": "coverImage" }, - "optional": true + "optional": false }, "date": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "title": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -8035,14 +8109,14 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "excerpt": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "featured": { "type": "objectAttribute", @@ -8933,21 +9007,21 @@ "type": "inline", "name": "coverImage" }, - "optional": true + "optional": false }, "date": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "title": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -8955,14 +9029,14 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "excerpt": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "featured": { "type": "objectAttribute", @@ -9786,21 +9860,21 @@ "type": "inline", "name": "coverImage" }, - "optional": true + "optional": false }, "date": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "title": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -9808,14 +9882,14 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "excerpt": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "featured": { "type": "objectAttribute", @@ -10639,7 +10713,7 @@ } } }, - "optional": true + "optional": false }, "sponsor": { "type": "objectAttribute", @@ -10677,6 +10751,28 @@ }, "optional": true }, + "syndications": { + "type": "objectAttribute", + "value": { + "type": "array", + "of": { + "type": "object", + "attributes": { + "_key": { + "type": "objectAttribute", + "value": { + "type": "string" + } + } + }, + "rest": { + "type": "inline", + "name": "syndication.reference" + } + } + }, + "optional": true + }, "statistics": { "type": "objectAttribute", "value": { @@ -10833,14 +10929,14 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "youtubeId": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "youtubeUrl": { "type": "objectAttribute", @@ -11164,21 +11260,21 @@ "type": "inline", "name": "coverImage" }, - "optional": true + "optional": false }, "date": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "title": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -11186,14 +11282,14 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "excerpt": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "featured": { "type": "objectAttribute", @@ -12017,7 +12113,7 @@ } } }, - "optional": true + "optional": false }, "sponsor": { "type": "objectAttribute", @@ -12055,6 +12151,28 @@ }, "optional": true }, + "syndications": { + "type": "objectAttribute", + "value": { + "type": "array", + "of": { + "type": "object", + "attributes": { + "_key": { + "type": "objectAttribute", + "value": { + "type": "string" + } + } + }, + "rest": { + "type": "inline", + "name": "syndication.reference" + } + } + }, + "optional": true + }, "statistics": { "type": "objectAttribute", "value": { @@ -12172,21 +12290,21 @@ "type": "inline", "name": "coverImage" }, - "optional": true + "optional": false }, "date": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "title": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "slug": { "type": "objectAttribute", @@ -12194,14 +12312,14 @@ "type": "inline", "name": "slug" }, - "optional": true + "optional": false }, "excerpt": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "featured": { "type": "objectAttribute", @@ -13025,7 +13143,7 @@ } } }, - "optional": true + "optional": false }, "sponsor": { "type": "objectAttribute", @@ -13063,6 +13181,28 @@ }, "optional": true }, + "syndications": { + "type": "objectAttribute", + "value": { + "type": "array", + "of": { + "type": "object", + "attributes": { + "_key": { + "type": "objectAttribute", + "value": { + "type": "string" + } + } + }, + "rest": { + "type": "inline", + "name": "syndication.reference" + } + } + }, + "optional": true + }, "statistics": { "type": "objectAttribute", "value": { @@ -13716,7 +13856,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "description": { "type": "objectAttribute", @@ -13805,7 +13945,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "_type": { "type": "objectAttribute", @@ -13872,14 +14012,14 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "path": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "sideOnly": { "type": "objectAttribute", @@ -14420,7 +14560,7 @@ "type": "inline", "name": "assist.instruction.context.reference" }, - "optional": true + "optional": false } } } @@ -14599,7 +14739,7 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "description": { "type": "objectAttribute", @@ -15069,21 +15209,21 @@ "value": { "type": "number" }, - "optional": true + "optional": false }, "width": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false }, "aspectRatio": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false } } } @@ -15238,35 +15378,35 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "extension": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "mimeType": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "size": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false }, "assetId": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "uploadId": { "type": "objectAttribute", @@ -15280,14 +15420,14 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "url": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "source": { "type": "objectAttribute", @@ -15411,35 +15551,35 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "extension": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "mimeType": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "size": { "type": "objectAttribute", "value": { "type": "number" }, - "optional": true + "optional": false }, "assetId": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "uploadId": { "type": "objectAttribute", @@ -15453,14 +15593,14 @@ "value": { "type": "string" }, - "optional": true + "optional": false }, "url": { "type": "objectAttribute", "value": { "type": "string" }, - "optional": true + "optional": false }, "metadata": { "type": "objectAttribute", diff --git a/apps/sanity/functions/sync-youtube-analytics/index.ts b/apps/sanity/functions/sync-youtube-analytics/index.ts new file mode 100644 index 000000000..04d73730b --- /dev/null +++ b/apps/sanity/functions/sync-youtube-analytics/index.ts @@ -0,0 +1,198 @@ +import { scheduledEventHandler } from "@sanity/functions"; +import { createClient } from "@sanity/client"; +import { youtubeParser } from "../../lib/utils"; + +interface TargetVideoDoc { + _id: string; + _type: string; + title?: string; + youtube: string; + transcript?: { + _type: "reference"; + _ref: string; + }; +} + +interface YouTubeVideoStatsItem { + id: string; + statistics?: { + viewCount?: string; + likeCount?: string; + commentCount?: string; + favoriteCount?: string; + }; +} + +export const handler = scheduledEventHandler(async ({ context }) => { + console.log( + "[Sync YouTube Analytics] Starting scheduled sync for YouTube analytics...", + ); + + const apiKey = process.env.YOUTUBE_API_KEY; + if (!apiKey) { + throw new Error( + "[Sync YouTube Analytics] YOUTUBE_API_KEY environment variable is missing.", + ); + } + + const token = + context.clientOptions?.token || + process.env.SANITY_API_WRITE_TOKEN || + process.env.SANITY_AUTH_TOKEN; + if (!token) { + throw new Error( + "[Sync YouTube Analytics] Missing Sanity API token (neither context.clientOptions.token nor SANITY_API_WRITE_TOKEN is present).", + ); + } + + const dataset = + process.env.SANITY_STUDIO_DATASET || + context.clientOptions?.dataset || + "production"; + + if (dataset !== "production") { + console.log( + `[Sync YouTube Analytics] Stack is "${dataset}". Nightly analytics sync only executes on production. Skipping.`, + ); + return; + } + + const projectId = + context.clientOptions?.projectId || + process.env.SANITY_STUDIO_PROJECT_ID || + "hfh83o0w"; + + const client = createClient({ + projectId, + dataset, + token, + apiVersion: "2025-09-30", + useCdn: false, + }); + + // Query all posts and podcasts with YouTube URLs + console.log( + `[Sync YouTube Analytics] Fetching documents with YouTube URLs in project ${projectId}, dataset ${dataset}...`, + ); + + const docs = await client.fetch( + `*[_type in ["post", "podcast"] && (defined(youtube) || defined(listenLinks.youtube)) && !(_id in path("drafts.**"))]{ + _id, + _type, + title, + "youtube": coalesce(youtube, listenLinks.youtube), + transcript + }`, + ); + + console.log( + `[Sync YouTube Analytics] Found ${docs.length} documents with YouTube URLs. Preparing batch lookups...`, + ); + + // Map documents by YouTube Video ID + const videoMap = new Map(); + for (const doc of docs) { + const videoId = youtubeParser(doc.youtube); + if (!videoId) continue; + const list = videoMap.get(videoId) || []; + list.push(doc); + videoMap.set(videoId, list); + } + + const allVideoIds = Array.from(videoMap.keys()); + console.log( + `[Sync YouTube Analytics] Unique YouTube Video IDs to query: ${allVideoIds.length}`, + ); + + const BATCH_SIZE = 50; // YouTube Data API v3 allows up to 50 IDs per request + let totalUpdated = 0; + + for (let i = 0; i < allVideoIds.length; i += BATCH_SIZE) { + const batchIds = allVideoIds.slice(i, i + BATCH_SIZE); + const idsParam = batchIds.join(","); + + console.log( + `[Sync YouTube Analytics] Querying batch ${Math.floor(i / BATCH_SIZE) + 1} (${batchIds.length} videos)...`, + ); + + try { + const apiUrl = `https://www.googleapis.com/youtube/v3/videos?part=statistics&id=${idsParam}&key=${apiKey}`; + const resp = await fetch(apiUrl); + + if (!resp.ok) { + const errorText = await resp.text(); + console.error( + `[Sync YouTube Analytics] YouTube API error (${resp.status}): ${errorText}`, + ); + continue; + } + + const data = (await resp.json()) as { items?: YouTubeVideoStatsItem[] }; + const items = data.items || []; + + for (const item of items) { + const videoId = item.id; + const stats = item.statistics; + if (!stats) continue; + + const parsedStats = { + viewCount: stats.viewCount ? parseInt(stats.viewCount, 10) : 0, + likeCount: stats.likeCount ? parseInt(stats.likeCount, 10) : 0, + commentCount: stats.commentCount + ? parseInt(stats.commentCount, 10) + : 0, + favoriteCount: stats.favoriteCount + ? parseInt(stats.favoriteCount, 10) + : 0, + }; + + const targetDocs = videoMap.get(videoId) || []; + for (const targetDoc of targetDocs) { + // 1. Update source document statistics + await client + .patch(targetDoc._id) + .set({ + statistics: { + youtube: parsedStats, + }, + }) + .commit() + .catch((err) => { + console.warn( + `[Sync YouTube Analytics] Failed to patch stats on ${targetDoc._id}:`, + err, + ); + }); + + // 2. Also update statistics on referenced transcript doc if present + const transcriptId = + targetDoc.transcript?._ref || `transcript-yt-${videoId}`; + await client + .patch(transcriptId) + .set({ + statistics: parsedStats, + lastFetchedAt: new Date().toISOString(), + }) + .commit() + .catch(() => { + // Ignore if transcript doc doesn't exist yet + }); + + totalUpdated++; + } + } + + // Small delay between batches to respect YouTube quota + await new Promise((resolve) => setTimeout(resolve, 500)); + } catch (err) { + console.error( + `[Sync YouTube Analytics] Failed processing batch starting at index ${i}:`, + err, + ); + } + } + + console.log( + `[Sync YouTube Analytics] Successfully completed YouTube analytics sync. Updated statistics on ${totalUpdated} references.`, + ); +}); diff --git a/apps/sanity/functions/sync-youtube-transcript/index.ts b/apps/sanity/functions/sync-youtube-transcript/index.ts index aaa4db046..6076a7495 100644 --- a/apps/sanity/functions/sync-youtube-transcript/index.ts +++ b/apps/sanity/functions/sync-youtube-transcript/index.ts @@ -322,13 +322,15 @@ export const handler = documentEventHandler( const transcriptId = `transcript-yt-${youtubeId}`; // Recursion guard and recent fetch check: - // If document already links to this transcript and the transcript was fetched within the last 24 hours, skip. const existingTranscript = await client - .getDocument<{ _id: string; lastFetchedAt?: string }>(transcriptId) + .getDocument<{ _id: string; status?: string; fullText?: string; cues?: any[]; lastFetchedAt?: string }>(transcriptId) .catch(() => null); const isLinked = rawData.transcript?._ref === transcriptId; + // If the transcript already exists and is marked completed with fullText, we don't re-scrape captions. + const isTranscriptCompleted = existingTranscript?.status === "completed" && Boolean(existingTranscript?.fullText); + if (existingTranscript && isLinked && existingTranscript.lastFetchedAt) { const lastFetched = new Date(existingTranscript.lastFetchedAt).getTime(); const now = Date.now(); @@ -374,17 +376,45 @@ export const handler = documentEventHandler( const durationSeconds = parseIsoDuration(duration); const chapters = extractChaptersFromDescription(snippet.description); - console.log( - `[Sync YouTube Transcript] Fetching timed captions for video ID: ${youtubeId}`, - ); - const captionResult = await fetchYouTubeCaptions(youtubeId); + let cues = existingTranscript?.cues || []; + let fullText = existingTranscript?.fullText || ""; + let status = existingTranscript?.status || "no_caption_available"; + + if (!isTranscriptCompleted) { + console.log( + `[Sync YouTube Transcript] Fetching timed captions for video ID: ${youtubeId}`, + ); + const captionResult = await fetchYouTubeCaptions(youtubeId); + cues = captionResult.cues; + fullText = captionResult.fullText; + status = captionResult.status; + } else { + console.log( + `[Sync YouTube Transcript] Transcript for ${youtubeId} already completed. Preserving cues/fullText and updating video metadata and statistics only.`, + ); + } const summary = generateSummary( snippet.title || rawData.title || "Video", snippet.description || "", - captionResult.fullText, + fullText, ); + const parsedStats = { + viewCount: statistics.viewCount + ? parseInt(statistics.viewCount, 10) + : undefined, + likeCount: statistics.likeCount + ? parseInt(statistics.likeCount, 10) + : undefined, + commentCount: statistics.commentCount + ? parseInt(statistics.commentCount, 10) + : undefined, + favoriteCount: statistics.favoriteCount + ? parseInt(statistics.favoriteCount, 10) + : undefined, + }; + // 1. Create or replace the published transcript document await client.createOrReplace({ _id: transcriptId, @@ -404,49 +434,38 @@ export const handler = documentEventHandler( description: snippet.description || "", tags: snippet.tags || [], topicCategories: topicDetails.topicCategories || [], - status: captionResult.status, - fullText: captionResult.fullText, + status, + fullText, summary, chapters, - cues: captionResult.cues, - statistics: { - viewCount: statistics.viewCount - ? parseInt(statistics.viewCount, 10) - : undefined, - likeCount: statistics.likeCount - ? parseInt(statistics.likeCount, 10) - : undefined, - commentCount: statistics.commentCount - ? parseInt(statistics.commentCount, 10) - : undefined, - favoriteCount: statistics.favoriteCount - ? parseInt(statistics.favoriteCount, 10) - : undefined, - }, + cues, + statistics: parsedStats, lastFetchedAt: new Date().toISOString(), }); console.log( - `[Sync YouTube Transcript] Successfully created/updated published transcript ${transcriptId} (cues: ${captionResult.cues.length}, status: ${captionResult.status}).`, + `[Sync YouTube Transcript] Successfully created/updated published transcript ${transcriptId} (cues: ${cues.length}, status: ${status}).`, ); - // 2. Link reference in parent document if not already linked + // 2. Patch parent document with transcript reference (if not linked) AND latest statistics.youtube + const patchPayload: Record = { + statistics: { + youtube: parsedStats, + }, + }; if (!isLinked) { - console.log( - `[Sync YouTube Transcript] Linking transcript ${transcriptId} to source document ${rawData._id}.`, - ); - await client - .patch(rawData._id) - .set({ - transcript: { - _type: "reference", - _ref: transcriptId, - }, - }) - .commit(); - console.log( - `[Sync YouTube Transcript] Parent document ${rawData._id} patched with transcript reference.`, - ); + patchPayload.transcript = { + _type: "reference", + _ref: transcriptId, + }; } + + console.log( + `[Sync YouTube Transcript] Patching parent document ${rawData._id} with statistics and transcript link.`, + ); + await client.patch(rawData._id).set(patchPayload).commit(); + console.log( + `[Sync YouTube Transcript] Parent document ${rawData._id} updated successfully.`, + ); }, ); diff --git a/apps/sanity/functions/syndicate-content/index.ts b/apps/sanity/functions/syndicate-content/index.ts index 4b86d2dc9..6d80f4404 100644 --- a/apps/sanity/functions/syndicate-content/index.ts +++ b/apps/sanity/functions/syndicate-content/index.ts @@ -16,6 +16,7 @@ interface ContentDocument { episode?: number; devto?: string; hashnode?: string; + syndications?: Array<{ _type: "reference"; _ref: string; _key?: string }>; } export const handler = documentEventHandler( @@ -36,8 +37,14 @@ export const handler = documentEventHandler( const isProduction = dataset === "production"; const projectId = context.clientOptions?.projectId || "hfh83o0w"; + const token = + context.clientOptions?.token || + process.env.SANITY_API_WRITE_TOKEN || + process.env.SANITY_AUTH_TOKEN; + const client = createClient({ ...context.clientOptions, + token, apiVersion: "2025-09-30", }); @@ -352,7 +359,7 @@ export const handler = documentEventHandler( if (existingHashnodeSlug && !existingHashnodeSlug.startsWith("simulated-")) { // Lookup post ID from Hashnode - const getPostRes = await fetch("https://gql.hashnode.com", { + const getPostRes = await fetch("https://gql-beta.hashnode.com", { method: "POST", headers: { "Content-Type": "application/json", @@ -378,7 +385,7 @@ export const handler = documentEventHandler( const updateInput = { ...articleInput }; delete updateInput.slug; // slug cannot be updated - const updateRes = await fetch("https://gql.hashnode.com", { + const updateRes = await fetch("https://gql-beta.hashnode.com", { method: "POST", headers: { "Content-Type": "application/json", @@ -412,7 +419,7 @@ export const handler = documentEventHandler( if (!publishedPost) { // Create post mutation - const publishRes = await fetch("https://gql.hashnode.com", { + const publishRes = await fetch("https://gql-beta.hashnode.com", { method: "POST", headers: { "Content-Type": "application/json", @@ -491,5 +498,35 @@ export const handler = documentEventHandler( }); } } + + // Update parent document's `syndications` reference array to maintain strong bidirectional links + try { + const expectedRefs = platforms.map((p) => `syndication.${data._id}.${p}`); + const existingRefs = (data.syndications || []).map((s) => s._ref); + + const newRefs = expectedRefs.filter((ref) => !existingRefs.includes(ref)); + if (newRefs.length > 0) { + const referencesToAppend = newRefs.map((ref) => ({ + _key: `syn_${ref.replace(/[^a-zA-Z0-9]/g, "_")}`, + _type: "reference" as const, + _ref: ref, + })); + + await client + .patch(data._id) + .setIfMissing({ syndications: [] }) + .append("syndications", referencesToAppend) + .commit(); + + console.log( + `[Syndicate Content] Linked ${newRefs.length} syndication references to parent ${data._id}.`, + ); + } + } catch (linkErr) { + console.warn( + `[Syndicate Content] Failed to link syndication references to parent ${data._id}:`, + linkErr, + ); + } }, ); diff --git a/apps/sanity/schemas/partials/content.ts b/apps/sanity/schemas/partials/content.ts index d72d99a27..3d292dd47 100644 --- a/apps/sanity/schemas/partials/content.ts +++ b/apps/sanity/schemas/partials/content.ts @@ -90,6 +90,15 @@ const content = defineType({ type: "string", group: "data", }), + defineField({ + name: "syndications", + title: "Syndications", + type: "array", + group: "data", + of: [{ type: "reference", to: [{ type: "syndication" }] }], + description: "Associated cross-platform syndication records (Dev.to, Hashnode, etc.)", + readOnly: true, + }), defineField({ name: "statistics", type: "object", diff --git a/sanity.blueprint.ts b/sanity.blueprint.ts index c75de2117..06784acb9 100644 --- a/sanity.blueprint.ts +++ b/sanity.blueprint.ts @@ -17,6 +17,13 @@ export default defineBlueprint({ src: "./apps/sanity/functions/syndicate-content", runtime: "nodejs22.x", timeout: 30, + env: { + DEVTO_API_KEY: process.env.PRIVATE_DEVTO || process.env.DEVTO_API_KEY || "", + HASHNODE_ACCESS_TOKEN: process.env.PRIVATE_HASHNODE || process.env.HASHNODE_ACCESS_TOKEN || "", + HASHNODE_PUBLICATION_ID: process.env.HASHNODE_PUBLICATION_ID || "60242f8180da6c44eadf775b", + SANITY_AUTH_TOKEN: process.env.SANITY_AUTH_TOKEN || "", + SANITY_API_WRITE_TOKEN: process.env.SANITY_API_WRITE_TOKEN || "", + }, }), defineDocumentFunction({ name: "sync-youtube-transcript", @@ -69,5 +76,24 @@ export default defineBlueprint({ process.env.SANITY_STUDIO_DATASET || "production", }, }), + defineScheduledFunction({ + name: "sync-youtube-analytics", + displayName: "Sync YouTube Video Analytics (Views, Likes, Comments)", + src: "./apps/sanity/functions/sync-youtube-analytics", + event: { + expression: "0 6 * * *", // 6:00 AM UTC + }, + runtime: "nodejs22.x", + timeout: 300, + env: { + YOUTUBE_API_KEY: process.env.YOUTUBE_API_KEY || "", + SANITY_AUTH_TOKEN: process.env.SANITY_AUTH_TOKEN || "", + SANITY_API_WRITE_TOKEN: process.env.SANITY_API_WRITE_TOKEN || "", + SANITY_STUDIO_PROJECT_ID: + process.env.SANITY_STUDIO_PROJECT_ID || "hfh83o0w", + SANITY_STUDIO_DATASET: + process.env.SANITY_STUDIO_DATASET || "production", + }, + }), ], }); diff --git a/scripts/backfill-content-from-transcripts.mjs b/scripts/backfill-content-from-transcripts.mjs new file mode 100644 index 000000000..3c4efa831 --- /dev/null +++ b/scripts/backfill-content-from-transcripts.mjs @@ -0,0 +1,182 @@ +import { createClient } from "@sanity/client"; + +const projectId = process.env.SANITY_STUDIO_PROJECT_ID || "hfh83o0w"; +const dataset = process.env.SANITY_STUDIO_DATASET || "production"; +const token = + process.env.SANITY_API_WRITE_TOKEN || process.env.SANITY_AUTH_TOKEN; + +if (!token) { + console.error( + "Error: Neither SANITY_API_WRITE_TOKEN nor SANITY_AUTH_TOKEN is defined in the environment.", + ); + process.exit(1); +} + +const client = createClient({ + projectId, + dataset, + token, + apiVersion: "2025-09-30", + useCdn: false, +}); + +function markdownToPortableText(markdownText) { + const lines = markdownText.split("\n"); + const blocks = []; + let keyCounter = 1; + const getKey = () => + `b_${Date.now().toString(36)}_${(keyCounter++).toString(36)}`; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i].trim(); + if (!line) continue; + + if (line.startsWith("### ")) { + blocks.push({ + _key: getKey(), + _type: "block", + style: "h3", + markDefs: [], + children: [ + { + _key: getKey(), + _type: "span", + marks: [], + text: line.replace(/^###\s+/, ""), + }, + ], + }); + } else if (line.startsWith("## ")) { + blocks.push({ + _key: getKey(), + _type: "block", + style: "h2", + markDefs: [], + children: [ + { + _key: getKey(), + _type: "span", + marks: [], + text: line.replace(/^##\s+/, ""), + }, + ], + }); + } else if (line.startsWith("> ")) { + blocks.push({ + _key: getKey(), + _type: "block", + style: "blockquote", + markDefs: [], + children: [ + { + _key: getKey(), + _type: "span", + marks: [], + text: line.replace(/^>\s+/, ""), + }, + ], + }); + } else if (line.startsWith("- ")) { + blocks.push({ + _key: getKey(), + _type: "block", + style: "normal", + listItem: "bullet", + level: 1, + markDefs: [], + children: [ + { + _key: getKey(), + _type: "span", + marks: [], + text: line.replace(/^-\s+/, ""), + }, + ], + }); + } else { + blocks.push({ + _key: getKey(), + _type: "block", + style: "normal", + markDefs: [], + children: [{ _key: getKey(), _type: "span", marks: [], text: line }], + }); + } + } + return blocks; +} + +// Curated high-fidelity content synthesized directly from full episode transcripts: + +const SVELTEKIT_DAISYUI_CONTENT = `## Overview: SvelteKit, DaisyUI, and Headless APIs +In this stream, Alex and Brittany dive into modern frontend development with SvelteKit, exploring how to build polished, responsive user interfaces without writing hundreds of lines of custom CSS. The discussion focuses on combining utility-first Tailwind CSS with the component-driven philosophy of DaisyUI, alongside integrating external dynamic content via Notion APIs. + +## Why DaisyUI with Tailwind CSS? +Tailwind CSS provides incredible design flexibility, but constructing common UI components—such as buttons, modals, dropdowns, and navigation bars—often requires long strings of utility classes. DaisyUI solves this by adding semantic class names (like \`btn\`, \`btn-primary\`, \`card\`, and \`navbar\`) directly to Tailwind CSS as a plugin. +- Pure CSS Components: DaisyUI components don't require external JavaScript runtime overhead. +- Built-in Theme Support: Instant light/dark and retro themes configurable via simple HTML \`data-theme\` attributes. +- Clean Svelte Markup: Allows Svelte components to remain concise and readable while retaining full Tailwind override capabilities. + +## Architecture & Code Concepts Explored +- SvelteKit Project Setup: Initializing a SvelteKit project and configuring \`tailwind.config.cjs\` with the DaisyUI plugin. +- Dynamic Component Styling: Using Svelte reactive props and template literals to conditionally apply DaisyUI variant classes (e.g., swapping button states dynamically). +- Data Ingestion & API Handling: Structuring server routes and environment variables in SvelteKit to securely query external endpoints. +- Layouts & Reusability: Creating reusable layout components and navigation wrappers across SvelteKit route hierarchies. + +## Key Takeaways +1. Semantic Tailwind: DaisyUI dramatically speeds up prototyping without locking you into rigid JavaScript UI frameworks. +2. SvelteKit Ergonomics: The intuitive separation of routing, server load functions, and client components in SvelteKit makes working with third-party CSS libraries seamless. +3. Rapid Theming: Switching whole application color palettes with DaisyUI themes takes just a single attribute change.`; + +const GITHUB_CMS_CONTENT = `## Overview: Using GitHub as a Headless Content Management System +In this live coding session, Alex explores treating GitHub repositories as a fully functional, open-source content management system (CMS) for modern web applications. By utilizing GitHub Discussions, Issues, Projects, and the GraphQL API, creators and developer communities can manage content openly without relying on proprietary databases or paid headless platforms. + +## The Motivation: Open Source & Community-Driven Content +Traditional CMS platforms often introduce separate authentication silos, proprietary editing interfaces, and recurring hosting costs. By contrast, leveraging GitHub as a CMS provides several distinct advantages: +- Transparent Contributions: Authors and readers already possess GitHub accounts and understand Markdown/Git workflows. +- Built-in Moderation & Version Control: Every edit, discussion thread, or revision history is preserved automatically with full cryptographic integrity. +- Zero Database Maintenance: GitHub stores and hosts the content while providing high-uptime GraphQL and REST APIs. + +## Architecture & Practical Implementation +- Authentication & Scoped Access: Generating GitHub Personal Access Tokens (PATs) with minimal scopes (\`read:org\`, \`repo\`) to safely query content. +- GraphQL API Integration: Constructing GitHub GraphQL queries to fetch project boards, milestone tasks, issue bodies, and discussion categories. +- Mapping Content to SvelteKit: Transforming fetched GitHub Markdown and frontmatter into structured data objects for rendering in SvelteKit pages. +- Sponsorships & Creator Compensation: Discussing models for using GitHub Sponsors to directly fund community authors and lesson creators. + +## Key Takeaways +1. Developer-Friendly Publishing: For developer-centric platforms and educational sites, GitHub represents a natural, accessible content authoring environment. +2. Structured Data from Projects: GitHub Projects (v2) with custom fields, statuses, and milestones provide enough metadata structure to power blogs, course outlines, and community roadmaps. +3. Openness by Default: Empowering users to submit improvements via pull requests or discussions turns a static blog into a collaborative knowledge base.`; + +async function run() { + console.log(`Target dataset: ${dataset}, project: ${projectId}`); + + // 1. SvelteKit with DaisyUI + const doc1Id = "52775db0-6f8e-4a54-9e7d-2910a96b662a"; + console.log(`Backfilling content for "${doc1Id}" (SvelteKit with DaisyUI)...`); + const blocks1 = markdownToPortableText(SVELTEKIT_DAISYUI_CONTENT); + await client + .patch(doc1Id) + .set({ content: blocks1 }) + .commit() + .then(() => console.log(`✓ Successfully updated ${doc1Id}`)) + .catch((err) => console.error(`✗ Error updating ${doc1Id}:`, err.message)); + + // 2. GitHub CMS + const doc2Id = "25596293-2ad4-47e5-b692-dbe6b8f80084"; + console.log(`Backfilling content for "${doc2Id}" (GitHub CMS)...`); + const blocks2 = markdownToPortableText(GITHUB_CMS_CONTENT); + await client + .patch(doc2Id) + .set({ content: blocks2 }) + .commit() + .then(() => console.log(`✓ Successfully updated ${doc2Id}`)) + .catch((err) => console.error(`✗ Error updating ${doc2Id}:`, err.message)); + + console.log("Backfill complete!"); +} + +run().catch((err) => { + console.error("Backfill script error:", err); + process.exit(1); +});