From 7292507bd4aaf36f29ec38fb7ec4e043ef492c96 Mon Sep 17 00:00:00 2001 From: bernardhanna Date: Thu, 16 Jul 2026 12:20:09 +0100 Subject: [PATCH 1/2] Read Excel hyperlink targets for Learn & Teach Link cells. Third-party rows store Articulate URLs as hyperlinks on title text; parsing display values alone left those resources without real links. Co-authored-by: Cursor --- app/Services/LearnTeachWorkbookParser.php | 7 ++++ scripts/prepare-learn-teach-upload.py | 40 ++++++++++++++--------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/app/Services/LearnTeachWorkbookParser.php b/app/Services/LearnTeachWorkbookParser.php index 698180d4d..1becb4b78 100644 --- a/app/Services/LearnTeachWorkbookParser.php +++ b/app/Services/LearnTeachWorkbookParser.php @@ -82,6 +82,13 @@ public function parse(string $path): array } $link = $cells[4] ?? ''; + $linkCell = $worksheet->getCell([4, $rowNum]); + if ($linkCell->hasHyperlink()) { + $target = trim((string) $linkCell->getHyperlink()->getUrl()); + if ($target !== '' && (str_starts_with($target, 'http://') || str_starts_with($target, 'https://'))) { + $link = $target; + } + } $image = $this->normalizeImage($cells[13] ?? ''); $rows[] = [ diff --git a/scripts/prepare-learn-teach-upload.py b/scripts/prepare-learn-teach-upload.py index 515f8f130..7df48aa65 100755 --- a/scripts/prepare-learn-teach-upload.py +++ b/scripts/prepare-learn-teach-upload.py @@ -125,7 +125,8 @@ def first_image_filename(image: str) -> str: def parse_workbook(xlsx_path: Path) -> list[dict]: - wb = openpyxl.load_workbook(xlsx_path, read_only=True) + # Must not use read_only/values_only — third-party Link cells store the real URL as a hyperlink. + wb = openpyxl.load_workbook(xlsx_path, data_only=False) rows_out: list[dict] = [] for sheet_name in wb.sheetnames: @@ -136,24 +137,31 @@ def parse_workbook(xlsx_path: Path) -> list[dict]: # Resource sheets: row 2 is an English group blurb (no language); data from row 3. # Third-party sheet: data starts at row 2. start_row = 2 if is_third_party else 3 - for row in ws.iter_rows(min_row=start_row, values_only=True): - if not any(c is not None and str(c).strip() for c in row): + for row_idx, row in enumerate(ws.iter_rows(min_row=start_row), start=start_row): + values = [c.value for c in row] + if not any(c is not None and str(c).strip() for c in values): continue - name = normalize_cell(row[2] if len(row) > 2 else "") - link_raw = normalize_cell(row[3] if len(row) > 3 else "") + name = normalize_cell(values[2] if len(values) > 2 else "") + link_raw = normalize_cell(values[3] if len(values) > 3 else "") if not name: continue + link_cell = row[3] if len(row) > 3 else None + if link_cell is not None and link_cell.hyperlink and link_cell.hyperlink.target: + target = str(link_cell.hyperlink.target).strip() + if target.startswith("http://") or target.startswith("https://"): + link_raw = target + if is_third_party: - lang = normalize_cell(row[11] if len(row) > 11 else "") + lang = normalize_cell(values[11] if len(values) > 11 else "") else: - lang = normalize_cell(row[1] if len(row) > 1 else "") + lang = normalize_cell(values[1] if len(values) > 1 else "") if not lang or lang.lower() == "language": continue import_link, source_url = link_to_filename(link_raw) - image_raw = normalize_cell(row[12] if len(row) > 12 else "") + image_raw = normalize_cell(values[12] if len(values) > 12 else "") image = first_image_filename(image_raw) if image_raw.startswith("http") and not image: image = image_raw.split(";")[0].strip() @@ -162,14 +170,14 @@ def parse_workbook(xlsx_path: Path) -> list[dict]: { "name_of_the_resource": name, "link": import_link, - "description": normalize_cell(row[4] if len(row) > 4 else ""), - "filters_type": normalize_cell(row[5] if len(row) > 5 else ""), - "filters_target_audience": normalize_cell(row[6] if len(row) > 6 else ""), - "filters_level_of_difficulty": normalize_cell(row[7] if len(row) > 7 else ""), - "filters_programming_language": normalize_cell(row[7 + 1] if len(row) > 7 + 1 else ""), - "filters_subject": normalize_cell(row[8 + 1] if len(row) > 8 + 1 else ""), - "filters_topics": normalize_cell(row[9 + 1] if len(row) > 9 + 1 else ""), - "filters_language": lang if not is_third_party else normalize_cell(row[11] if len(row) > 11 else ""), + "description": normalize_cell(values[4] if len(values) > 4 else ""), + "filters_type": normalize_cell(values[5] if len(values) > 5 else ""), + "filters_target_audience": normalize_cell(values[6] if len(values) > 6 else ""), + "filters_level_of_difficulty": normalize_cell(values[7] if len(values) > 7 else ""), + "filters_programming_language": normalize_cell(values[8] if len(values) > 8 else ""), + "filters_subject": normalize_cell(values[9] if len(values) > 9 else ""), + "filters_topics": normalize_cell(values[10] if len(values) > 10 else ""), + "filters_language": lang if not is_third_party else normalize_cell(values[11] if len(values) > 11 else ""), "category": "", "group_name": group if not is_third_party else "Third party resources", "image": image, From 4ecc15bb77cbda42e43fd29647eb2ceb1d7cf453 Mon Sep 17 00:00:00 2001 From: bernardhanna Date: Thu, 16 Jul 2026 12:24:49 +0100 Subject: [PATCH 2/2] Add internal ops docs for Learn & Teach resource imports. Start a tracked docs/ops area for site runbooks, beginning with the SharePoint-to-S3 localisation upload workflow. Co-authored-by: Cursor --- docs/ops/README.md | 12 + docs/ops/learn-and-teach-resource-import.md | 259 ++++++++++++++++++++ 2 files changed, 271 insertions(+) create mode 100644 docs/ops/README.md create mode 100644 docs/ops/learn-and-teach-resource-import.md diff --git a/docs/ops/README.md b/docs/ops/README.md new file mode 100644 index 000000000..c5f3f633e --- /dev/null +++ b/docs/ops/README.md @@ -0,0 +1,12 @@ +# Internal ops docs + +Runbooks and notes for operating the Code Week site (deploy, imports, content publishing). + +These live under `docs/ops/` so they are **tracked in git**. +Do **not** put secrets, credentials, or large binaries here. + +Private working files (spreadsheets, PDFs, ZIPs) stay in `docs/internal/` (gitignored). + +| Doc | Purpose | +|-----|---------| +| [Learn & Teach resource import](./learn-and-teach-resource-import.md) | Bulk upload localisation PDFs + third-party links to `/resources/learn-and-teach` | diff --git a/docs/ops/learn-and-teach-resource-import.md b/docs/ops/learn-and-teach-resource-import.md new file mode 100644 index 000000000..17043c46c --- /dev/null +++ b/docs/ops/learn-and-teach-resource-import.md @@ -0,0 +1,259 @@ +# Learn & Teach resource import + +Internal runbook for publishing resources to **Learn & Teach** +(`https://codeweek.eu/resources/learn-and-teach`). + +Use this when the content team delivers a multi-sheet Excel workbook (metadata) plus PDFs/thumbnails (often on SharePoint). + +--- + +## What gets published + +| Kind | What it is | How `source` works | +|------|------------|--------------------| +| **Localised PDFs** | One resource per language (e.g. Albanian *Kodo një Mik*) | PDF uploaded to S3 (`RESOURCES_BUCKET`); card opens that PDF | +| **Third-party links** | External courses/sites (Intel Articulate, Science-on-Stage, etc.) | `source` is an `https://…` URL; card opens it in a **new tab** | + +Frontend: `ResourceCard` uses `resource.source` with `target="_blank"`. + +Language filter: each localised row is tagged with `filters_language` (e.g. Albanian). Users filter via the **Languages** control on the page — not via the site locale switcher (Shqip / English in the header). + +--- + +## Tools in this repo + +| Piece | Role | +|-------|------| +| `php artisan resources:import {file}` | CLI import: reads CSV/Excel next to `images/` + `links/`, uploads to S3, creates/updates `ResourceItem` | +| `/admin/resources-import` | Super-admin UI (verify → preview → import). Supports multi-sheet Learn & Teach workbooks + optional assets ZIP (after `learn-tech-import` is merged) | +| `scripts/prepare-learn-teach-upload.py` | Flattens multi-sheet Excel → `metadata.csv` + folder layout; copies from a local SharePoint sync folder | +| `scripts/download-learn-teach-from-sharepoint.py` | Downloads PDFs/thumbnails via **Microsoft Graph** using `az login` | +| `App\Services\LearnTeachWorkbookParser` | Parses the multi-sheet workbook; **reads Excel hyperlink targets** on Link cells | +| `App\Services\SharePointAssetFetcher` | Optional: fetch SharePoint URLs during import (only if the server can access the link) | + +S3 config (server `.env`): + +- `RESOURCES_BUCKET` / `RESOURCES_URL` +- `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_DEFAULT_REGION` + +No extra AWS form is needed in the admin UI — imports use that disk (`resources`). + +--- + +## Spreadsheet format + +Typical delivery: one Excel file with: + +- **One sheet per resource group** (e.g. `Code a Friend`, `Binary Number Challenge`) +- Row 1 = headers +- Row 2 = English group blurb (no language) — **skipped** by the parser +- Row 3+ = one row per language +- Final sheet: **`Third party resources`** + +Important columns (display names → import keys): + +| Spreadsheet | Import key | +|-------------|------------| +| Name of the Resource | `name_of_the_resource` | +| Link | `link` | +| Description | `description` | +| Filters - TYPE | `filters_type` | +| Filters - TARGET AUDIENCE | `filters_target_audience` | +| Filters - LEVEL OF DIFFICULTY | `filters_level_of_difficulty` | +| Filters - PROGRAMMING LANGUAGE | `filters_programming_language` | +| Filters - SUBJECT | `filters_subject` | +| Filters - TOPICS | `filters_topics` | +| Filters - LANGUAGE / Language | `filters_language` | +| Image | `image` | + +The import also sets `group_name` from the **sheet title** (used to find PDFs under `links/{group_name}/`). + +### Link column gotchas + +1. **PDF filename** (most localisation rows): e.g. `ALBANIAN_02 Code a Friend_LOCALISED.pdf` + → must exist under `links/{group}/` (or be downloadable). +2. **SharePoint URL** (sometimes): full `https://….sharepoint.com/…pdf` + → download locally first, or use Graph script / guest-accessible fetch. +3. **Excel hyperlink on title text** (Intel third-party): cell *displays* the course title but the real URL is the hyperlink (e.g. `https://share.articulate.com/…`). + **Always read the hyperlink target**, not only the visible value. + `LearnTeachWorkbookParser` and `prepare-learn-teach-upload.py` do this. + +### Image column + +Local filename (`Binary number challenge.png`) → file in `images/`. +Or an `https://…` URL (kept / fetched when possible). + +--- + +## Folder layout for CLI import + +``` +learn-teach-upload-YYYY/ +├── metadata.csv # flattened rows +├── images/ # thumbnails +└── links/ + ├── Binary Number Challenge/ + │ └── ALBANIAN_01 ….pdf + ├── Code a Friend/ + │ └── … + └── … +``` + +The Excel/CSV path and the `images` / `links` folders must sit in the **same directory**. + +```bash +php artisan resources:import /path/to/learn-teach-upload-YYYY/metadata.csv --focus --batch-timestamp +``` + +Useful flags: + +| Flag | Meaning | +|------|---------| +| `--focus` | Create missing filter options (types, levels, languages, …) | +| `--batch-timestamp` | Same Unix suffix for all S3 files in this run | +| `--stable-names` | No timestamp (overwrites same S3 key) | +| `--preserve-filenames` | Use local basenames as S3 keys | + +--- + +## Recommended workflow + +### 1. Get PDFs from SharePoint (authenticated) + +SharePoint is not anonymously downloadable. Use Azure CLI against the JA Europe tenant, then Graph: + +```bash +az login --tenant dc75c1d7-1a32-4e97-af90-1dc01911ea6c --allow-no-subscriptions + +python3 scripts/download-learn-teach-from-sharepoint.py \ + --xlsx "docs/internal/New Resources Learn&Teach Metadata - 23 June 2026.xlsx" \ + --out docs/internal/learn-teach-upload-2026 + +# Optional: one language only +python3 scripts/download-learn-teach-from-sharepoint.py --languages Albanian +``` + +Then rebuild metadata (and optionally re-copy from a sync folder): + +```bash +python3 scripts/prepare-learn-teach-upload.py \ + --xlsx "docs/internal/New Resources Learn&Teach Metadata - 23 June 2026.xlsx" \ + --out docs/internal/learn-teach-upload-2026 \ + --source-dir docs/internal/learn-teach-upload-2026 +``` + +Working assets stay under **`docs/internal/`** (gitignored). Do **not** commit PDFs/ZIPs. + +Zip for admin UI upload if needed: + +```bash +cd docs/internal/learn-teach-upload-2026 +zip -r learn-teach-assets.zip links images +``` + +### 2. Test on DEV first + +Servers (Forge): + +| Env | SSH | App dir | +|-----|-----|---------| +| **DEV** | `ssh -i ~/.ssh/id_rsa forge@35.156.58.10` | `cd dev.codeweek.eu` | +| **Live** | `ssh -i ~/.ssh/id_rsa forge@3.68.107.5` | `cd codeweek.eu` | + +```bash +# From your laptop: upload package +scp -i ~/.ssh/id_rsa metadata.csv learn-teach-assets.zip \ + forge@35.156.58.10:/home/forge/dev.codeweek.eu/storage/app/learn-teach-upload-2026/ + +# On DEV — unzip + import in the SAME session +ssh -i ~/.ssh/id_rsa forge@35.156.58.10 +cd /home/forge/dev.codeweek.eu +cd storage/app/learn-teach-upload-2026 && unzip -o -q learn-teach-assets.zip +cd /home/forge/dev.codeweek.eu +php artisan resources:import "$(realpath storage/app/learn-teach-upload-2026/metadata.csv)" \ + --focus --batch-timestamp +``` + +**Note:** DEV often uses the same `RESOURCES_BUCKET` as production (`codeweek-resources`). PDFs land in that bucket; DB rows are only on the DEV database until you import on live. + +### 3. QA on DEV + +1. Open https://dev.codeweek.eu/resources/learn-and-teach +2. Hard refresh (`Cmd+Shift+R`) +3. **Languages** filter → Albanian (or another language) — expect localised titles +4. Open **View lesson** → correct language PDF from S3 +5. Spot-check a third-party card → Articulate / external site in a new tab + +If the language filter shows **no cards** after selecting a language while you were on page 2+, you need the ResourceForm fix (filters must reset to page 1). That fix lives on branch `learn-tech-import`. + +### 4. Publish on live + +Same upload + unzip + `resources:import` on `codeweek.eu`, then QA on +https://codeweek.eu/resources/learn-and-teach + +Prefer **one SSH session** for unzip + import so assets are present on the same host that runs Artisan. + +### 5. Git / deploy flow for code changes + +Feature work (importer, scripts, UI fixes): + +1. Branch (e.g. `learn-tech-import`) +2. PR → `dev` +3. QA on DEV +4. PR → `main` / `master` + +Do not push one-off PDF packages to git. + +--- + +## Admin UI path (when deployed) + +1. Sign in as **super admin** +2. `/admin/resources-import` +3. Upload multi-sheet `.xlsx` **or** `metadata.csv` +4. Optionally upload assets ZIP (`links/` + `images/`) +5. Enable **Focus** → **Verify** → edit Link/Image in preview if needed → **Import** + +Third-party hyperlinks are resolved by `LearnTeachWorkbookParser` when uploading the raw workbook. + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---------|--------------|-----| +| Card link is a PDF **filename**, not S3 URL | `links/` missing or wrong path at import time | Re-upload assets; import with absolute path to CSV in same dir as `links/` | +| Intel courses open nothing / show title as URL | Excel hyperlink not read | Re-parse with hyperlink-aware parser; update `source` to Articulate URL | +| Language filter returns empty | UI stayed on page 2+ after selecting language | Deploy ResourceForm fix; or click Search / go to page 1 | +| SharePoint download fails (login HTML) | No auth / wrong tenant | `az login` to JA Europe tenant; use Graph download script | +| Filter values missing | Typo vs DB (`beginners`, `Ukranian`, …) | Import with `--focus` or normalise spreadsheet | +| English “Uploaded on …” date looks old | Description text from an earlier English upload | Localisation batch often has no English rows; update English description separately if needed | + +Check recent rows: + +```bash +php artisan tinker --execute=" +\$n = \\App\\ResourceItem::where('updated_at', '>=', now()->subHour())->count(); +echo \"updated last hour: \$n\n\"; +" +``` + +--- + +## June 2026 batch (reference) + +- Workbook: `docs/internal/New Resources Learn&Teach Metadata - 23 June 2026.xlsx` +- ~12 resource groups × 17 languages ≈ **204 PDFs** +- **16** third-party rows (12 Articulate + others with plain URLs) +- Package dir used: `docs/internal/learn-teach-upload-2026/` +- Branch with importer/UI/script improvements: `learn-tech-import` + +--- + +## Related code + +- `app/Imports/ResourcesImport.php` +- `app/Console/Commands/ImportResourcesFromExcel.php` +- `app/Http/Controllers/ResourcesImportController.php` +- `resources/js/components/ResourceForm.vue` / `ResourceCard.vue` +- `config/filesystems.php` → `disks.resources`