Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
version: 2
updates:
- package-ecosystem: pip
- package-ecosystem: uv
directory: /
cooldown:
default-days: 7
semver-major-days: 30
schedule:
interval: monthly
groups:
all:
python-dependencies:
patterns:
- "*"
commit-message:
Expand All @@ -21,8 +21,21 @@ updates:
schedule:
interval: monthly
groups:
all:
github-actions:
patterns:
- "*"
commit-message:
prefix: "ci(deps):"

- package-ecosystem: pre-commit
directory: /
cooldown:
default-days: 7
schedule:
interval: monthly
groups:
pre-commit:
patterns:
- "*"
commit-message:
prefix: "chore(deps):"
43 changes: 18 additions & 25 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,31 +10,24 @@ permissions:
contents: read

jobs:
test:
ci:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- name: test
run: uv run --locked --group dev pytest
- name: lint
run: uv run --locked --group dev pre-commit run --all-files
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup Python
uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0
with:
python-version: "3.11"
cache: "pip"
- name: Install dependencies
run: pip install -r requirements.txt
- name: Test collections
run: pytest
lint:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup Python
uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

- name: Install uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
python-version: "3.11"
cache: "pip"
- name: Install dependencies
run: pip install -r requirements.txt
- name: Run pre-commit hooks
run: pre-commit run --all-files
enable-cache: true

- name: ${{ matrix.name }}
run: ${{ matrix.run }}
41 changes: 19 additions & 22 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,15 @@ jobs:
contents: read
issues: write
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0

- name: Install uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true

# Initializes the PR comment
# Edits existing or creates new comment
# Why? - Cleanliness!
Expand Down Expand Up @@ -121,7 +126,7 @@ jobs:
AIRFLOW_JWT_SECRET: ${{ secrets.AIRFLOW_JWT_SECRET }}
AIRFLOW_JWT_SUB: ${{ secrets.AIRFLOW_JWT_SUB }}
run: |
pip install -r ./scripts/requirements.txt
uv sync --locked --group scripts

# Track successful publications
all_failed=true
Expand All @@ -148,7 +153,7 @@ jobs:

echo "Publishing $config_type: $collection_id"

response=$(python3 "$script_to_run" "$file" "staging")
response=$(uv run --locked --group scripts python "$script_to_run" "$file" "staging")
echo "Processed file: $file"
status_code=$(echo "$response" | jq -r '.statusCode' | head -n1)
echo "Status Code: $status_code"
Expand Down Expand Up @@ -185,15 +190,6 @@ jobs:
$status_message"
gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"

- name: Set up Python
uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
with:
python-version: '3.9'
- uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
with:
path: ${{ env.pythonLocation }}
key: ${{ env.pythonLocation }}-pip-${{ hashFiles('requirements.txt') }}

# If the workflow fails at any point, the PR comment will be updated
- name: Update PR comment on overall workflow failure
if: failure()
Expand Down Expand Up @@ -238,7 +234,7 @@ jobs:
run: ls -lah /tmp/upload-artifacts

- name: Upload published collections files as directory artifact
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
env:
ARTIFACT_RETENTION_DAYS: ${{ vars.ARTIFACT_RETENTION_DAYS }}
with:
Expand All @@ -252,7 +248,7 @@ jobs:
echo "${{ github.run_id }}" > run_id.txt

- name: Upload Workflow Run ID
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
env:
ARTIFACT_RETENTION_DAYS: ${{ vars.ARTIFACT_RETENTION_DAYS }}
with:
Expand All @@ -268,7 +264,6 @@ jobs:
--name collections-to-promote-from-${{ github.event.pull_request.number }} \
--dir ./downloaded-files


mdx-pr-flag-evaluator:
runs-on: ubuntu-latest
environment: staging
Expand All @@ -277,7 +272,7 @@ jobs:
skip_mdx_pr_step: ${{ steps.skip_mdx_pr_step_setting.outputs.skip_mdx_pr_step }}
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

- name: Set SKIP_MDX_PR_STEP condition
id: skip_mdx_pr_step_setting
Expand All @@ -298,7 +293,12 @@ jobs:
pull-requests: write
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

- name: Install uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true

- name: Use output from publish-new-datasets
run: |
Expand All @@ -311,9 +311,9 @@ jobs:
run: |
echo $PUBLISHED_COLLECTION_FILES
collection_ids=""
pip install -r ./scripts/requirements.txt
uv sync --locked --group scripts
for file in ${PUBLISHED_COLLECTION_FILES}; do
collection_id=$(python3 ./scripts/generate_mdx.py "$file")
collection_id=$(uv run --locked --group scripts python ./scripts/generate_mdx.py "$file")
collection_id=$(echo "$collection_id" | sed 's/^["\s]*//;s/["\s]*$//')
echo "Processed collection ID: $collection_id"
collection_ids="$collection_ids$collection_id,"
Expand All @@ -323,9 +323,6 @@ jobs:
echo "Final collection_ids: $collection_ids"
echo "collection_ids=${collection_ids}" >> $GITHUB_ENV




- name: Debug - Print files to promote
run: |
ls downloaded-files
Expand Down
11 changes: 8 additions & 3 deletions .github/workflows/promote.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,12 @@ jobs:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

- name: Install uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true

- name: Determine artifact name
run: echo "ARTIFACT_NAME=collections-to-promote-from-${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV
Expand All @@ -72,7 +77,7 @@ jobs:
AIRFLOW_JWT_SUB: ${{ secrets.AIRFLOW_JWT_SUB }}

run: |
pip install -r ./scripts/requirements.txt
uv sync --locked --group scripts
for file in downloaded-files/*.json; do
echo "Processing $file"
# Check if collection key exists to determine if it's a dataset or collection
Expand All @@ -87,7 +92,7 @@ jobs:
fi

echo "Promoting $config_type ($collection_id) to production"
response=$(python3 "$script_to_run" "$file" "production")
response=$(uv run --locked --group scripts python "$script_to_run" "$file" "production")

echo "Processed file: $file"
status_code=$(echo "$response" | jq -r '.statusCode')
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/promotion-checker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
contents: read
steps:
- name: Checkout Repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

- name: Get Workflow Run ID Artifact
run: |
Expand Down
8 changes: 0 additions & 8 deletions .markdownlint-cli2.jsonc

This file was deleted.

35 changes: 23 additions & 12 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,22 +1,33 @@
exclude: ^transformation-scripts/.*$

repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
rev: v6.0.0
hooks:
- id: check-yaml
- id: file-contents-sorter
files: ^requirements.in$
- id: trailing-whitespace
exclude: ^ingestion-data/.*$

- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.16.1
hooks:
- id: ruff-check
args: [--fix]
- id: ruff-format
types_or: [python, pyi, pyproject]

- repo: https://github.com/DavidAnson/markdownlint-cli2
rev: v0.10.0
rev: v0.23.2
hooks:
- id: markdownlint-cli2
- repo: https://github.com/psf/black-pre-commit-mirror
rev: 24.3.0
hooks:
- id: black
- id: black-jupyter
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.0.292
args:
- "--fix"
- "--config"
- "pyproject.toml"
- "--configPointer"
- "/tool/markdownlint-cli2"

- repo: https://github.com/astral-sh/uv-pre-commit
rev: 0.12.5
hooks:
- id: ruff
- id: uv-lock
6 changes: 1 addition & 5 deletions CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1,5 +1 @@
* @anayeaye
* @gadomski
* @ividito
* @slesaad
* @smohiudd
* @NASA-IMPACT/veda-data-services

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🙌

2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ To add data to VEDA you will:

### Step 1: Stage your files

Upload files to the staging bucket `s3://veda-data-store-staging` (which you can do with a VEDA JupyterHub account--request access [here](https://docs.openveda.cloud/user-guide/scientific-computing/getting-access.html)) or a self-hosted bucket in s3 has shared read access to VEDA service. [See docs.openveda.cloud for additional details on preparing files.](https://docs.openveda.cloud/user-guide/content-curation/dataset-ingestion/file-preparation.html)
Upload files to the staging bucket `s3://veda-data-store-staging` (which you can do with a VEDA JupyterHub account--request [access here](https://docs.openveda.cloud/user-guide/scientific-computing/getting-access.html)) or a self-hosted bucket in s3 has shared read access to VEDA service. [See docs.openveda.cloud for additional details on preparing files.](https://docs.openveda.cloud/user-guide/content-curation/dataset-ingestion/file-preparation.html)

### Step 2: Generate STAC metadata in the staging catalog

Expand Down
56 changes: 56 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
[project]
name = "veda-data"
version = "0.1.0"
description = "Config data used to load datasets into the VEDA catalog"
readme = "README.md"
requires-python = ">=3.11"
dependencies = []

[dependency-groups]
dev = [
"pre-commit==4.6.1",
"pystac[validation]==1.8.4",
"pytest==9.0.3",
]
scripts = [
"pyyaml",
"PyJWT",
]

[tool.uv]
default-groups = ["dev"]

[tool.ruff]
line-length = 88

[tool.ruff.lint]
select = [
"E", # pycodestyle errors
"W", # pycodestyle warnings
"F", # Pyflakes
"I", # isort
"UP", # pyupgrade
"B", # flake8-bugbear
"C4", # flake8-comprehensions
"SIM", # flake8-simplify
"PTH", # flake8-use-pathlib
"RET", # flake8-return
"TID", # flake8-tidy-imports
"TC", # flake8-type-checking
"ISC", # flake8-implicit-str-concat
"ICN", # flake8-import-conventions
]
ignore = [
"BLE001", # Blind Exceptions
]

[tool.ruff.format]
docstring-code-format = true
docstring-code-line-length = "dynamic"

[tool.markdownlint-cli2.config]
MD013 = false
MD033 = { allowed_elements = ["a", "b", "details", "summary"] }

[tool.ruff.lint.isort]
combine-as-imports = true
6 changes: 0 additions & 6 deletions requirements.in

This file was deleted.

Loading
Loading