Platform NDJSON autosplit (#23990)
Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
parent
6f04278edb
commit
5ad4cc717b
107
.github/workflows/ci.yml
vendored
107
.github/workflows/ci.yml
vendored
@ -18,10 +18,6 @@ on:
|
||||
- cron: "0 8 * * *" # runs at 08:00 UTC every day
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
hub:
|
||||
description: "Run HUB"
|
||||
default: true
|
||||
type: boolean
|
||||
benchmarks:
|
||||
description: "Run Benchmarks"
|
||||
default: true
|
||||
@ -48,63 +44,6 @@ on:
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
HUB:
|
||||
if: github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.hub == 'true'))
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
python: ["3.12"]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-python@v6
|
||||
if: matrix.os != 'cpu-latest'
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
activate-environment: ${{ matrix.os == 'cpu-latest' }}
|
||||
- name: Install requirements
|
||||
shell: bash # for Windows compatibility
|
||||
run: |
|
||||
uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} . --extra-index-url https://download.pytorch.org/whl/cpu
|
||||
- name: Check environment
|
||||
run: |
|
||||
yolo checks
|
||||
uv pip list
|
||||
- name: Test HUB training
|
||||
shell: python
|
||||
env:
|
||||
API_KEY: ${{ secrets.ULTRALYTICS_HUB_API_KEY }}
|
||||
MODEL_ID: ${{ secrets.ULTRALYTICS_HUB_MODEL_ID }}
|
||||
run: |
|
||||
import os
|
||||
from ultralytics import YOLO, hub
|
||||
api_key, model_id = os.environ['API_KEY'], os.environ['MODEL_ID']
|
||||
hub.login(api_key)
|
||||
hub.reset_model(model_id)
|
||||
model = YOLO('https://hub.ultralytics.com/models/' + model_id)
|
||||
model.train()
|
||||
- name: Test HUB inference API
|
||||
shell: python
|
||||
env:
|
||||
API_KEY: ${{ secrets.ULTRALYTICS_HUB_API_KEY }}
|
||||
MODEL_ID: ${{ secrets.ULTRALYTICS_HUB_MODEL_ID }}
|
||||
run: |
|
||||
import os
|
||||
import requests
|
||||
import json
|
||||
api_key, model_id = os.environ['API_KEY'], os.environ['MODEL_ID']
|
||||
url = f"https://api.ultralytics.com/v1/predict/{model_id}"
|
||||
headers = {"x-api-key": api_key}
|
||||
data = {"size": 320, "confidence": 0.25, "iou": 0.45}
|
||||
with open("ultralytics/assets/zidane.jpg", "rb") as f:
|
||||
response = requests.post(url, headers=headers, data=data, files={"image": f})
|
||||
assert response.status_code == 200, f'Status code {response.status_code}, Reason {response.reason}'
|
||||
print(json.dumps(response.json(), indent=2))
|
||||
|
||||
Benchmarks:
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.benchmarks == 'true'
|
||||
runs-on: ${{ matrix.os }}
|
||||
@ -228,79 +167,79 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [cpu-latest, macos-26, windows-latest, ubuntu-24.04-arm]
|
||||
os: [ubuntu-latest, macos-26, windows-latest, ubuntu-24.04-arm]
|
||||
python: ["3.12"]
|
||||
torch: [latest]
|
||||
include:
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.8" # torch 1.8.0 requires python >=3.6, <=3.9
|
||||
torch: "1.8.0"
|
||||
torchvision: "0.9.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.9"
|
||||
torch: "1.9.0"
|
||||
torchvision: "0.10.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.9"
|
||||
torch: "1.10.0"
|
||||
torchvision: "0.11.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.10"
|
||||
torch: "1.11.0"
|
||||
torchvision: "0.12.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.10"
|
||||
torch: "1.12.0"
|
||||
torchvision: "0.13.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.10"
|
||||
torch: "1.13.0"
|
||||
torchvision: "0.14.0"
|
||||
- os: cpu-latest # Axelera exports
|
||||
- os: ubuntu-latest # Axelera exports
|
||||
python: "3.10"
|
||||
torch: "2.8.0"
|
||||
torchvision: "0.23.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.11"
|
||||
torch: "2.0.0"
|
||||
torchvision: "0.15.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.11"
|
||||
torch: "2.1.0"
|
||||
torchvision: "0.16.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.12"
|
||||
torch: "2.2.0"
|
||||
torchvision: "0.17.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.12"
|
||||
torch: "2.3.0"
|
||||
torchvision: "0.18.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.12"
|
||||
torch: "2.4.0"
|
||||
torchvision: "0.19.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.12"
|
||||
torch: "2.5.0"
|
||||
torchvision: "0.20.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.12"
|
||||
torch: "2.6.0"
|
||||
torchvision: "0.21.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.12"
|
||||
torch: "2.7.0"
|
||||
torchvision: "0.22.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.12"
|
||||
torch: "2.8.0"
|
||||
torchvision: "0.23.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.12"
|
||||
torch: "2.9.0"
|
||||
torchvision: "0.24.0"
|
||||
- os: cpu-latest
|
||||
- os: ubuntu-latest
|
||||
python: "3.12"
|
||||
torch: "2.10.0"
|
||||
torchvision: "0.25.0"
|
||||
@ -347,7 +286,7 @@ jobs:
|
||||
activate-environment: true
|
||||
- name: Install requirements
|
||||
run: |
|
||||
uv pip install -e . pytest-cov nvidia-ml-py
|
||||
uv pip install -e . pytest-cov nvidia-ml-py "torch<2.11" # TODO: remove torch pin after GPU runner driver update to CUDA 13.0
|
||||
env:
|
||||
PIP_BREAK_SYSTEM_PACKAGES: 1
|
||||
- name: Check environment
|
||||
@ -541,12 +480,12 @@ jobs:
|
||||
# run: pytest ultralytics/tests/test_python.py -vv -s
|
||||
|
||||
Summary:
|
||||
runs-on: cpu-latest
|
||||
needs: [HUB, Benchmarks, Tests, SlowTests, GPU, RaspberryPi, NVIDIA_Jetson, Conda]
|
||||
runs-on: ubuntu-latest
|
||||
needs: [Benchmarks, Tests, SlowTests, GPU, RaspberryPi, NVIDIA_Jetson, Conda]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Check for failure and notify
|
||||
if: (needs.HUB.result == 'failure' || needs.Benchmarks.result == 'failure' || needs.Tests.result == 'failure' || needs.SlowTests.result == 'failure' || needs.GPU.result == 'failure' || needs.RaspberryPi.result == 'failure' || needs.NVIDIA_Jetson.result == 'failure' || needs.Conda.result == 'failure' ) && github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event_name == 'push') && github.run_attempt == '1'
|
||||
if: (needs.Benchmarks.result == 'failure' || needs.Tests.result == 'failure' || needs.SlowTests.result == 'failure' || needs.GPU.result == 'failure' || needs.RaspberryPi.result == 'failure' || needs.NVIDIA_Jetson.result == 'failure' || needs.Conda.result == 'failure' ) && github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event_name == 'push') && github.run_attempt == '1'
|
||||
uses: slackapi/slack-github-action@v3.0.1
|
||||
with:
|
||||
webhook-type: incoming-webhook
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
||||
|
||||
__version__ = "8.4.25"
|
||||
__version__ = "8.4.26"
|
||||
|
||||
import importlib
|
||||
import os
|
||||
|
||||
@ -858,7 +858,22 @@ async def convert_ndjson_to_yolo(ndjson_path: str | Path, output_path: str | Pat
|
||||
if "train" not in splits:
|
||||
raise ValueError(f"Dataset missing required 'train' split. Found splits: {sorted(splits)}")
|
||||
if "val" not in splits and "test" not in splits:
|
||||
raise ValueError(f"Dataset missing required 'val' split. Found splits: {sorted(splits)}")
|
||||
train_records = [r for r in image_records if r.get("split") == "train"]
|
||||
if len(train_records) < 2:
|
||||
raise ValueError(
|
||||
f"Dataset has only {len(train_records)} image(s) and no 'val' split. "
|
||||
f"Need at least 2 images to auto-split into train/val."
|
||||
)
|
||||
random.Random(0).shuffle(train_records) # local RNG to avoid mutating global training seed
|
||||
val_count = max(1, len(train_records) // 10)
|
||||
for r in train_records[:val_count]:
|
||||
r["split"] = "val"
|
||||
splits.add("val")
|
||||
LOGGER.warning(
|
||||
f"WARNING ⚠️ No 'val' split found in dataset. "
|
||||
f"Auto-splitting {len(train_records)} images into {len(train_records) - val_count} train, {val_count} val. "
|
||||
f"For best results, manually assign validation images in Platform dataset page."
|
||||
)
|
||||
if task == "pose" and "kpt_shape" not in dataset_record:
|
||||
dataset_record["kpt_shape"] = _infer_ndjson_kpt_shape(image_records)
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ import socket
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from time import time
|
||||
from time import sleep, time
|
||||
|
||||
from ultralytics.utils import ENVIRONMENT, GIT, LOGGER, PYTHON_VERSION, RANK, SETTINGS, TESTS_RUNNING, Retry, colorstr
|
||||
|
||||
@ -88,37 +88,46 @@ def resolve_platform_uri(uri, hard=True):
|
||||
else:
|
||||
raise ValueError(f"Invalid platform URI: {uri}. Use ul://user/datasets/name or ul://user/project/model")
|
||||
|
||||
# (connect_timeout, read_timeout) — short connect so retries are fast, long read for server-side generation
|
||||
timeout = (10, 3600) if "/datasets/" in url else (10, 90)
|
||||
|
||||
try:
|
||||
timeout = 3600 if "/datasets/" in url else 90 # NDJSON generation can be slow for large datasets
|
||||
r = requests.head(url, headers=headers, allow_redirects=False, timeout=timeout)
|
||||
|
||||
# Handle redirect responses (301, 302, 303, 307, 308)
|
||||
if 300 <= r.status_code < 400 and "location" in r.headers:
|
||||
return r.headers["location"] # Return signed URL
|
||||
|
||||
# Handle error responses
|
||||
if r.status_code == 401:
|
||||
raise ValueError(f"Invalid ULTRALYTICS_API_KEY for '{uri}'")
|
||||
if r.status_code == 403:
|
||||
raise PermissionError(f"Access denied for '{uri}'. Check dataset/model visibility settings.")
|
||||
if r.status_code == 404:
|
||||
if hard:
|
||||
raise FileNotFoundError(f"Not found on platform: {uri}")
|
||||
LOGGER.warning(f"Not found on platform: {uri}")
|
||||
return None
|
||||
if r.status_code == 409:
|
||||
raise RuntimeError(f"Resource not ready: {uri}. Dataset may still be processing.")
|
||||
|
||||
# Unexpected response
|
||||
r.raise_for_status()
|
||||
raise RuntimeError(f"Unexpected response from platform for '{uri}': {r.status_code}")
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
for attempt in range(3):
|
||||
try:
|
||||
r = requests.head(url, headers=headers, allow_redirects=False, timeout=timeout)
|
||||
break
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
LOGGER.warning(f"Retry {attempt + 1}/3 failed for {uri}: {e}")
|
||||
if attempt >= 2:
|
||||
raise
|
||||
sleep(2 * (2**attempt)) # 2s, 4s backoff
|
||||
except Exception as e:
|
||||
if hard:
|
||||
raise ConnectionError(f"Failed to resolve {uri}: {e}") from e
|
||||
LOGGER.warning(f"Failed to resolve {uri}: {e}")
|
||||
return None
|
||||
|
||||
# Handle redirect responses (301, 302, 303, 307, 308)
|
||||
if 300 <= r.status_code < 400 and "location" in r.headers:
|
||||
return r.headers["location"] # Return signed URL
|
||||
|
||||
# Handle error responses
|
||||
if r.status_code == 401:
|
||||
raise ValueError(f"Invalid ULTRALYTICS_API_KEY for '{uri}'")
|
||||
if r.status_code == 403:
|
||||
raise PermissionError(f"Access denied for '{uri}'. Check dataset/model visibility settings.")
|
||||
if r.status_code == 404:
|
||||
if hard:
|
||||
raise FileNotFoundError(f"Not found on platform: {uri}")
|
||||
LOGGER.warning(f"Not found on platform: {uri}")
|
||||
return None
|
||||
if r.status_code == 409:
|
||||
raise RuntimeError(f"Resource not ready: {uri}. Dataset may still be processing.")
|
||||
|
||||
# Unexpected response
|
||||
r.raise_for_status()
|
||||
raise RuntimeError(f"Unexpected response from platform for '{uri}': {r.status_code}")
|
||||
|
||||
|
||||
def _interp_plot(plot, n=101):
|
||||
"""Interpolate plot curve data to n points to reduce storage size."""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user