droboticflow/src/services/api/object-storage.ts
2026-08-19 03:08:37 +08:00

419 lines
14 KiB
TypeScript

import type {
ObjectStorageBrowseResult,
ObjectStorageObjectDeleteResult,
ObjectStorageObjectMutationResult,
ObjectStorageOverview,
} from "../../types";
import { parseObjectStoragePreviewBytesHeader } from "../../lib/object-storage-preview.js";
import {
MULTIPART_CONCURRENCY,
MULTIPART_PART_BYTES,
PRESIGN_EXPIRES_SECONDS,
objectStorageMultipartParts,
objectStorageUploadMode,
parseObjectStorageUploadETag,
uploadObjectStoragePresignedURL,
} from "../../lib/object-storage-upload.js";
import { buildAPIURL } from "./base-url.js";
import { apiFetch, apiRequest } from "./client";
function buildObjectStorageParams(options?: {
connectionId?: string;
bucket?: string;
prefix?: string;
key?: string;
contentType?: string;
transcode?: string;
recursive?: boolean;
preview?: boolean;
marker?: string;
limit?: number;
}) {
const params = new URLSearchParams();
if (options?.connectionId?.trim()) {
params.set("connectionId", options.connectionId.trim());
}
if (options?.bucket?.trim()) {
params.set("bucket", options.bucket.trim());
}
if (typeof options?.prefix === "string") {
params.set("prefix", options.prefix.trim());
}
if (options?.key?.trim()) {
params.set("key", options.key.trim());
}
if (options?.contentType?.trim()) {
params.set("contentType", options.contentType.trim());
}
if (options?.transcode?.trim()) {
params.set("transcode", options.transcode.trim());
}
if (options?.recursive) {
params.set("recursive", "true");
}
if (options?.preview) {
params.set("preview", "true");
}
if (typeof options?.marker === "string") {
params.set("marker", options.marker.trim());
}
if (typeof options?.limit === "number" && Number.isInteger(options.limit) && options.limit > 0) {
params.set("limit", String(options.limit));
}
const query = params.toString();
return query ? `?${query}` : "";
}
const OBJECT_STORAGE_PREVIEW_TRUNCATED_HEADER = "X-Drobodata-Preview-Truncated";
const OBJECT_STORAGE_PREVIEW_ORIGINAL_SIZE_HEADER = "X-Drobodata-Preview-Original-Bytes";
const OBJECT_STORAGE_PREVIEW_RETURNED_SIZE_HEADER = "X-Drobodata-Preview-Returned-Bytes";
export interface ObjectStorageTextPreviewResult {
content: string;
truncated: boolean;
originalSizeBytes?: number;
returnedSizeBytes?: number;
}
export type ObjectStorageUploadMode = "direct" | "multipart";
export interface ObjectStorageUploadProgress {
mode: ObjectStorageUploadMode;
loadedBytes: number;
totalBytes: number;
partNumber?: number;
partCount?: number;
}
export type ObjectStorageBatchConflictPolicy = "overwrite" | "skip" | "error";
export type ObjectStorageBatchPreparedAction = "upload" | "create_placeholder" | "skip" | "conflict";
export interface ObjectStorageBatchUploadItemInput {
relativePath: string;
kind: "file" | "directory";
sizeBytes?: number;
contentType?: string;
}
export interface ObjectStorageBatchUploadPreparedItem extends ObjectStorageBatchUploadItemInput {
key: string;
action: ObjectStorageBatchPreparedAction;
exists: boolean;
message?: string;
}
export interface ObjectStorageBatchUploadPrepareResult {
bucket: string;
prefix?: string;
conflictPolicy: ObjectStorageBatchConflictPolicy;
items: ObjectStorageBatchUploadPreparedItem[];
uploadCount: number;
placeholderCount: number;
skipCount: number;
conflictCount: number;
}
interface ObjectStoragePresignedUploadResult {
bucket: string;
key: string;
method: "PUT";
url: string;
headers?: Record<string, string>;
expiresAt: string;
}
interface ObjectStorageMultipartUploadCreateResult {
bucket: string;
key: string;
uploadId: string;
expiresAt: string;
}
interface ObjectStorageMultipartUploadPartURLResult {
bucket: string;
key: string;
uploadId: string;
partNumber: number;
method: "PUT";
url: string;
headers?: Record<string, string>;
expiresAt: string;
}
interface ObjectStorageMultipartUploadPart {
partNumber: number;
eTag: string;
}
async function uploadWithFormData<T>(path: string, formData: FormData) {
const response = await apiFetch(
path,
{
method: "POST",
body: formData,
},
{ includeJSONContentType: false },
);
if (!response.ok) {
const text = await response.text();
throw new Error(text || `Request failed: ${response.status}`);
}
return (await response.json()) as T;
}
async function uploadMultipartObject(
file: File,
options: { connectionId: string; bucket?: string; key: string; contentType?: string },
onProgress?: (progress: ObjectStorageUploadProgress) => void,
) {
const initiated = await apiRequest<ObjectStorageMultipartUploadCreateResult>(
`/api/object-storage/uploads/multipart/initiate${buildObjectStorageParams({ connectionId: options.connectionId })}`,
{ method: "POST" },
{
bucket: options.bucket,
key: options.key,
contentType: options.contentType,
expiresInSeconds: PRESIGN_EXPIRES_SECONDS,
},
);
const parts = objectStorageMultipartParts(file.size, MULTIPART_PART_BYTES);
const uploadedParts: ObjectStorageMultipartUploadPart[] = [];
let completedBytes = 0;
let nextIndex = 0;
let aborted = false;
const activePartBytes = new Map<number, number>();
const reportMultipartProgress = (partNumber?: number) => {
const activeBytes = Array.from(activePartBytes.values()).reduce((total, loadedBytes) => total + loadedBytes, 0);
onProgress?.({
mode: "multipart",
loadedBytes: Math.min(completedBytes + activeBytes, file.size),
totalBytes: file.size,
partNumber,
partCount: parts.length,
});
};
const abortUpload = async () => {
if (aborted) {
return;
}
aborted = true;
try {
await apiRequest(
`/api/object-storage/uploads/multipart/abort${buildObjectStorageParams({ connectionId: options.connectionId })}`,
{ method: "POST" },
{ bucket: options.bucket, key: options.key, uploadId: initiated.uploadId },
);
} catch {
// Preserve the original upload failure.
}
};
const uploadNext = async () => {
while (nextIndex < parts.length) {
const part = parts[nextIndex];
nextIndex += 1;
const partURL = await apiRequest<ObjectStorageMultipartUploadPartURLResult>(
`/api/object-storage/uploads/multipart/part-url${buildObjectStorageParams({ connectionId: options.connectionId })}`,
{ method: "POST" },
{
bucket: options.bucket,
key: options.key,
uploadId: initiated.uploadId,
partNumber: part.partNumber,
expiresInSeconds: PRESIGN_EXPIRES_SECONDS,
},
);
const partSize = part.end - part.start;
activePartBytes.set(part.partNumber, 0);
const response = await uploadObjectStoragePresignedURL(
partURL.url,
file.slice(part.start, part.end),
partURL.headers,
(loadedBytes) => {
activePartBytes.set(part.partNumber, Math.min(loadedBytes, partSize));
reportMultipartProgress(part.partNumber);
},
);
const eTag = parseObjectStorageUploadETag(response.headers.get("ETag"));
if (!eTag) {
throw new Error(`Object storage upload part ${part.partNumber} did not return ETag`);
}
uploadedParts.push({ partNumber: part.partNumber, eTag });
activePartBytes.delete(part.partNumber);
completedBytes += partSize;
reportMultipartProgress(part.partNumber);
}
};
try {
onProgress?.({ mode: "multipart", loadedBytes: 0, totalBytes: file.size, partCount: parts.length });
await Promise.all(Array.from({ length: Math.min(MULTIPART_CONCURRENCY, parts.length) }, () => uploadNext()));
uploadedParts.sort((left, right) => left.partNumber - right.partNumber);
return await apiRequest<ObjectStorageObjectMutationResult>(
`/api/object-storage/uploads/multipart/complete${buildObjectStorageParams({ connectionId: options.connectionId })}`,
{ method: "POST" },
{ bucket: options.bucket, key: options.key, uploadId: initiated.uploadId, parts: uploadedParts },
);
} catch (error) {
await abortUpload();
throw error;
}
}
async function uploadDirectObject(
file: File,
options: { connectionId: string; bucket?: string; key: string; contentType?: string },
onProgress?: (progress: ObjectStorageUploadProgress) => void,
) {
const presigned = await apiRequest<ObjectStoragePresignedUploadResult>(
`/api/object-storage/uploads/presign-put${buildObjectStorageParams({ connectionId: options.connectionId })}`,
{ method: "POST" },
{
bucket: options.bucket,
key: options.key,
contentType: options.contentType,
expiresInSeconds: PRESIGN_EXPIRES_SECONDS,
},
);
onProgress?.({ mode: "direct", loadedBytes: 0, totalBytes: file.size });
await uploadObjectStoragePresignedURL(presigned.url, file, presigned.headers, (loadedBytes) => {
onProgress?.({
mode: "direct",
loadedBytes: Math.min(loadedBytes, file.size),
totalBytes: file.size,
});
});
onProgress?.({ mode: "direct", loadedBytes: file.size, totalBytes: file.size });
const prefix = options.key.includes("/") ? options.key.slice(0, options.key.lastIndexOf("/")) : "";
const listed = await apiRequest<ObjectStorageBrowseResult>(
`/api/object-storage/objects${buildObjectStorageParams({
connectionId: options.connectionId,
bucket: options.bucket,
prefix,
})}`,
);
const object = listed.items.find((item) => item.key === options.key);
if (!object) {
throw new Error("Object storage upload completed but uploaded object was not found");
}
return { object };
}
export const objectStorageApi = {
getOverview: (options?: { connectionId?: string }) =>
apiRequest<ObjectStorageOverview>(`/api/object-storage${buildObjectStorageParams(options)}`),
listObjects: (options: { connectionId: string; bucket?: string; prefix?: string; marker?: string; limit?: number }) =>
apiRequest<ObjectStorageBrowseResult>(`/api/object-storage/objects${buildObjectStorageParams(options)}`),
putObject: (
payload: { bucket?: string; key: string; content: string; contentType?: string },
options: { connectionId: string },
) =>
apiRequest<ObjectStorageObjectMutationResult>(
`/api/object-storage/objects${buildObjectStorageParams(options)}`,
{ method: "POST" },
payload,
),
uploadObject: async (
file: File,
options: { connectionId: string; bucket?: string; key: string; contentType?: string },
onProgress?: (progress: ObjectStorageUploadProgress) => void,
) => {
if (objectStorageUploadMode(file.size) === "multipart") {
return uploadMultipartObject(file, options, onProgress);
}
return uploadDirectObject(file, options, onProgress);
},
prepareBatchUpload: (
payload: {
bucket?: string;
prefix?: string;
conflictPolicy: ObjectStorageBatchConflictPolicy;
items: ObjectStorageBatchUploadItemInput[];
},
options: { connectionId: string },
) =>
apiRequest<ObjectStorageBatchUploadPrepareResult>(
`/api/object-storage/uploads/batch/prepare${buildObjectStorageParams(options)}`,
{ method: "POST" },
payload,
),
uploadObjectWithServerFallback: async (
file: File,
options: { connectionId: string; bucket?: string; key: string; contentType?: string },
) => {
const formData = new FormData();
formData.append("file", file);
return uploadWithFormData<ObjectStorageObjectMutationResult>(
`/api/object-storage/objects/upload${buildObjectStorageParams(options)}`,
formData,
);
},
renameObject: (
payload: { bucket?: string; key: string; newKey: string; recursive?: boolean },
options: { connectionId: string },
) =>
apiRequest<ObjectStorageObjectMutationResult>(
`/api/object-storage/objects${buildObjectStorageParams(options)}`,
{ method: "PATCH" },
payload,
),
moveObjects: (
payload: {
bucket?: string;
destinationPrefix: string;
items: Array<{ key: string; kind?: string; recursive?: boolean }>;
},
options: { connectionId: string },
) =>
apiRequest<ObjectStorageObjectMutationResult>(
`/api/object-storage/objects${buildObjectStorageParams(options)}`,
{ method: "PATCH" },
payload,
),
deleteObject: (options: { connectionId: string; bucket?: string; key: string; recursive?: boolean }) =>
apiRequest<ObjectStorageObjectDeleteResult>(`/api/object-storage/objects${buildObjectStorageParams(options)}`, {
method: "DELETE",
}),
restoreObject: (
payload: { bucket?: string; trashKey: string; originalKey: string; recursive?: boolean },
options: { connectionId: string },
) =>
apiRequest<ObjectStorageObjectMutationResult>(
`/api/object-storage/objects/restore${buildObjectStorageParams(options)}`,
{ method: "POST" },
payload,
),
buildContentURL: (options: { connectionId: string; bucket?: string; key: string; transcode?: string }) =>
buildAPIURL(`/api/object-storage/content${buildObjectStorageParams(options)}`),
readObjectTextPreview: async (options: {
connectionId: string;
bucket?: string;
key: string;
}): Promise<ObjectStorageTextPreviewResult> => {
const response = await apiFetch(
`/api/object-storage/content${buildObjectStorageParams({
...options,
preview: true,
})}`,
undefined,
{ includeJSONContentType: false },
);
if (!response.ok) {
const text = await response.text();
throw new Error(text || `Request failed: ${response.status}`);
}
const content = await response.text();
const parseBytesHeader = (headerName: string) => {
return parseObjectStoragePreviewBytesHeader(response.headers.get(headerName));
};
return {
content,
truncated: response.headers.get(OBJECT_STORAGE_PREVIEW_TRUNCATED_HEADER) === "true",
originalSizeBytes: parseBytesHeader(OBJECT_STORAGE_PREVIEW_ORIGINAL_SIZE_HEADER),
returnedSizeBytes: parseBytesHeader(OBJECT_STORAGE_PREVIEW_RETURNED_SIZE_HEADER),
};
},
};