dataSync.yml

name: Data Sync

# ──────────────────────────────────────────────────────────────
# DART 메인 데이터 수집 (2026-04-16 정리)
#
# 자동 cron (recent 모드):
#   1. DART list.json 으로 최근 N일 정기공시 조회
#   2. 카테고리별 독립 누락 검사 (docs rcept_no + finance reprt_code)
#   3. 누락된 종목·분기만 수집 — 88분기 차집합 X
#   4. 이전 실행 pending.txt (API 한도 잘림) 우선 회수
#   5. 변경 파일만 HF push
#
# 수동 (workflow_dispatch):
#   - mode=recent (기본): list.json 기반 경량 수집 (~30분)
#   - mode=full: 88분기 차집합 수집 (~5시간, 이전 dataPipeline full 모드 대체)
#
# 파이프라인 상세: ops/data.md
# ──────────────────────────────────────────────────────────────

on:
  schedule:
    - cron: '0 18 * * *'  # UTC 18:00 = KST 03:00 (야간 공시 캡처)
    - cron: '0 6 * * *'   # UTC 06:00 = KST 15:00 (당일 오전 공시 빠른 반영)
  workflow_dispatch:
    inputs:
      lookback_days:
        description: '신규 공시 조회 기간 (일)'
        default: '60'
        type: string
      mode:
        description: 'recent (list.json 기반) / full (88분기 차집합, 무거움)'
        default: 'recent'
        type: choice
        options: [recent, full]
      category:
        description: '(full 모드용) finance/report/docs/all'
        default: 'all'
        type: choice
        options: [all, finance, report, docs]

permissions:
  contents: write

# 같은 HF dataset repo (eddmpython/dartlab-data) 에 push 하는 모든 워크플로우는
# 공용 concurrency group 으로 직렬화 — HF sliding-window 429 구조적 차단.
concurrency:
  group: hf-dataset-push
  cancel-in-progress: false

jobs:
  # ── Job 1: finance + report (list.json 기반, ~30분) ──
  sync-finance-report:
    if: ${{ github.event.inputs.mode != 'full' }}
    runs-on: ubuntu-latest
    timeout-minutes: 60

    steps:
      - uses: actions/checkout@v5

      - name: Install uv
        uses: astral-sh/setup-uv@v5

      - name: Set up Python
        run: uv python install 3.12

      - name: Install dependencies
        run: uv sync

      - name: Restore finance cache
        uses: actions/cache/restore@v5
        with:
          path: data/dart/finance
          key: dartlab-data-finance-${{ github.run_id }}
          restore-keys: dartlab-data-finance-

      - name: Restore report cache
        uses: actions/cache/restore@v5
        with:
          path: data/dart/report
          key: dartlab-data-report-${{ github.run_id }}
          restore-keys: dartlab-data-report-

      - name: Restore docs cache (비교용)
        uses: actions/cache/restore@v5
        with:
          path: data/dart/docs
          key: dartlab-data-docs-${{ github.run_id }}
          restore-keys: dartlab-data-docs-

      - name: Restore collect state (pending.txt) — fr scope
        uses: actions/cache/restore@v5
        with:
          path: data/dart/_collect_state/fr
          key: dartlab-collect-state-fr-${{ github.run_id }}
          restore-keys: dartlab-collect-state-fr-

      - name: Sync recent (finance + report)
        run: uv run python .github/scripts/syncRecent.py
        env:
          DART_API_KEYS: ${{ secrets.DART_API_KEYS }}
          SYNC_LOOKBACK_DAYS: ${{ github.event.inputs.lookback_days || '60' }}
          SYNC_CATEGORIES: finance,report
          SYNC_STATE_SCOPE: fr  # 병렬 Job pending.txt 경쟁 회피

      - name: Upload to HuggingFace (finance)
        if: hashFiles('dist/changed_finance.txt') != ''
        run: |
          if [ -s dist/changed_finance.txt ]; then
            cp dist/changed_finance.txt dist/changed.txt
            uv run python .github/scripts/uploadData.py --target hf
          fi
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          SYNC_CATEGORY: finance

      - name: Upload to HuggingFace (report)
        if: hashFiles('dist/changed_report.txt') != ''
        run: |
          if [ -s dist/changed_report.txt ]; then
            cp dist/changed_report.txt dist/changed.txt
            uv run python .github/scripts/uploadData.py --target hf
          fi
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          SYNC_CATEGORY: report

      # GitHub Releases 업로드 폐지 (2026-04-08): HF만 유지.
      # data-finance/data-report 태그는 정리됨. uploadData.py --target gh 호출 금지.

      - name: Save finance cache
        uses: actions/cache/save@v5
        with:
          path: data/dart/finance
          key: dartlab-data-finance-${{ github.run_id }}

      - name: Save report cache
        uses: actions/cache/save@v5
        with:
          path: data/dart/report
          key: dartlab-data-report-${{ github.run_id }}

      - name: Save collect state — fr scope
        uses: actions/cache/save@v5
        with:
          path: data/dart/_collect_state/fr
          key: dartlab-collect-state-fr-${{ github.run_id }}

  # ── Job 2: docs (느림, 별도 타임아웃) ──
  sync-docs:
    if: ${{ github.event.inputs.mode != 'full' }}
    runs-on: ubuntu-latest
    timeout-minutes: 150

    steps:
      - uses: actions/checkout@v5

      - name: Install uv
        uses: astral-sh/setup-uv@v5

      - name: Set up Python
        run: uv python install 3.12

      - name: Install dependencies
        run: uv sync

      - name: Restore docs cache
        uses: actions/cache/restore@v5
        with:
          path: data/dart/docs
          key: dartlab-data-docs-${{ github.run_id }}
          restore-keys: dartlab-data-docs-

      - name: Restore collect state (pending.txt) — docs scope
        uses: actions/cache/restore@v5
        with:
          path: data/dart/_collect_state/docs
          key: dartlab-collect-state-docs-${{ github.run_id }}
          restore-keys: dartlab-collect-state-docs-

      - name: Sync recent (docs)
        run: uv run python .github/scripts/syncRecent.py
        env:
          DART_API_KEYS: ${{ secrets.DART_API_KEYS }}
          SYNC_LOOKBACK_DAYS: ${{ github.event.inputs.lookback_days || '60' }}
          SYNC_CATEGORIES: docs
          SYNC_STATE_SCOPE: docs

      - name: Upload to HuggingFace (docs)
        if: hashFiles('dist/changed_docs.txt') != ''
        run: |
          if [ -s dist/changed_docs.txt ]; then
            cp dist/changed_docs.txt dist/changed.txt
            uv run python .github/scripts/uploadData.py --target hf
          fi
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          SYNC_CATEGORY: docs

      # GitHub Releases 업로드 폐지 (2026-04-08): HF만 유지.

      - name: Save docs cache
        uses: actions/cache/save@v5
        with:
          path: data/dart/docs
          key: dartlab-data-docs-${{ github.run_id }}

      - name: Save collect state — docs scope
        uses: actions/cache/save@v5
        with:
          path: data/dart/_collect_state/docs
          key: dartlab-collect-state-docs-${{ github.run_id }}

  # ── Job 3 (workflow_dispatch full 전용): 88분기 차집합 (heavy) ──
  full-collect:
    if: ${{ github.event.inputs.mode == 'full' }}
    runs-on: ubuntu-latest
    timeout-minutes: 300

    strategy:
      fail-fast: false
      matrix:
        category: ${{ github.event_name == 'workflow_dispatch' && fromJSON(format('["{0}"]', inputs.category == 'all' && 'finance","report","docs' || inputs.category)) || fromJSON('["finance","report","docs"]') }}

    steps:
      - uses: actions/checkout@v5

      - name: Install uv
        uses: astral-sh/setup-uv@v5

      - name: Set up Python
        run: uv python install 3.12

      - name: Install dependencies
        run: uv sync

      - name: Restore data cache
        uses: actions/cache/restore@v5
        with:
          path: data/dart/${{ matrix.category }}
          key: dartlab-data-${{ matrix.category }}-${{ github.run_id }}
          restore-keys: dartlab-data-${{ matrix.category }}-

      - name: Full collect (88-quarter diff)
        run: uv run python .github/scripts/syncData.py
        env:
          DART_API_KEYS: ${{ secrets.DART_API_KEYS }}
          SYNC_CATEGORY: ${{ matrix.category }}
          SYNC_MODE: all

      - name: Save data cache
        uses: actions/cache/save@v5
        with:
          path: data/dart/${{ matrix.category }}
          key: dartlab-data-${{ matrix.category }}-${{ github.run_id }}

      - name: Upload to HuggingFace
        run: uv run python .github/scripts/uploadData.py --target hf
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          SYNC_CATEGORY: ${{ matrix.category }}

      # GitHub Releases 업로드 폐지 (2026-04-08): HF만 유지.