forked from longtao.wu/droboticflow
419 lines
14 KiB
TypeScript
419 lines
14 KiB
TypeScript
import type {
|
|
ObjectStorageBrowseResult,
|
|
ObjectStorageObjectDeleteResult,
|
|
ObjectStorageObjectMutationResult,
|
|
ObjectStorageOverview,
|
|
} from "../../types";
|
|
import { parseObjectStoragePreviewBytesHeader } from "../../lib/object-storage-preview.js";
|
|
import {
|
|
MULTIPART_CONCURRENCY,
|
|
MULTIPART_PART_BYTES,
|
|
PRESIGN_EXPIRES_SECONDS,
|
|
objectStorageMultipartParts,
|
|
objectStorageUploadMode,
|
|
parseObjectStorageUploadETag,
|
|
uploadObjectStoragePresignedURL,
|
|
} from "../../lib/object-storage-upload.js";
|
|
import { buildAPIURL } from "./base-url.js";
|
|
import { apiFetch, apiRequest } from "./client";
|
|
|
|
function buildObjectStorageParams(options?: {
|
|
connectionId?: string;
|
|
bucket?: string;
|
|
prefix?: string;
|
|
key?: string;
|
|
contentType?: string;
|
|
transcode?: string;
|
|
recursive?: boolean;
|
|
preview?: boolean;
|
|
marker?: string;
|
|
limit?: number;
|
|
}) {
|
|
const params = new URLSearchParams();
|
|
if (options?.connectionId?.trim()) {
|
|
params.set("connectionId", options.connectionId.trim());
|
|
}
|
|
if (options?.bucket?.trim()) {
|
|
params.set("bucket", options.bucket.trim());
|
|
}
|
|
if (typeof options?.prefix === "string") {
|
|
params.set("prefix", options.prefix.trim());
|
|
}
|
|
if (options?.key?.trim()) {
|
|
params.set("key", options.key.trim());
|
|
}
|
|
if (options?.contentType?.trim()) {
|
|
params.set("contentType", options.contentType.trim());
|
|
}
|
|
if (options?.transcode?.trim()) {
|
|
params.set("transcode", options.transcode.trim());
|
|
}
|
|
if (options?.recursive) {
|
|
params.set("recursive", "true");
|
|
}
|
|
if (options?.preview) {
|
|
params.set("preview", "true");
|
|
}
|
|
if (typeof options?.marker === "string") {
|
|
params.set("marker", options.marker.trim());
|
|
}
|
|
if (typeof options?.limit === "number" && Number.isInteger(options.limit) && options.limit > 0) {
|
|
params.set("limit", String(options.limit));
|
|
}
|
|
const query = params.toString();
|
|
return query ? `?${query}` : "";
|
|
}
|
|
|
|
const OBJECT_STORAGE_PREVIEW_TRUNCATED_HEADER = "X-Drobodata-Preview-Truncated";
|
|
const OBJECT_STORAGE_PREVIEW_ORIGINAL_SIZE_HEADER = "X-Drobodata-Preview-Original-Bytes";
|
|
const OBJECT_STORAGE_PREVIEW_RETURNED_SIZE_HEADER = "X-Drobodata-Preview-Returned-Bytes";
|
|
|
|
export interface ObjectStorageTextPreviewResult {
|
|
content: string;
|
|
truncated: boolean;
|
|
originalSizeBytes?: number;
|
|
returnedSizeBytes?: number;
|
|
}
|
|
|
|
export type ObjectStorageUploadMode = "direct" | "multipart";
|
|
|
|
export interface ObjectStorageUploadProgress {
|
|
mode: ObjectStorageUploadMode;
|
|
loadedBytes: number;
|
|
totalBytes: number;
|
|
partNumber?: number;
|
|
partCount?: number;
|
|
}
|
|
|
|
export type ObjectStorageBatchConflictPolicy = "overwrite" | "skip" | "error";
|
|
export type ObjectStorageBatchPreparedAction = "upload" | "create_placeholder" | "skip" | "conflict";
|
|
|
|
export interface ObjectStorageBatchUploadItemInput {
|
|
relativePath: string;
|
|
kind: "file" | "directory";
|
|
sizeBytes?: number;
|
|
contentType?: string;
|
|
}
|
|
|
|
export interface ObjectStorageBatchUploadPreparedItem extends ObjectStorageBatchUploadItemInput {
|
|
key: string;
|
|
action: ObjectStorageBatchPreparedAction;
|
|
exists: boolean;
|
|
message?: string;
|
|
}
|
|
|
|
export interface ObjectStorageBatchUploadPrepareResult {
|
|
bucket: string;
|
|
prefix?: string;
|
|
conflictPolicy: ObjectStorageBatchConflictPolicy;
|
|
items: ObjectStorageBatchUploadPreparedItem[];
|
|
uploadCount: number;
|
|
placeholderCount: number;
|
|
skipCount: number;
|
|
conflictCount: number;
|
|
}
|
|
|
|
interface ObjectStoragePresignedUploadResult {
|
|
bucket: string;
|
|
key: string;
|
|
method: "PUT";
|
|
url: string;
|
|
headers?: Record<string, string>;
|
|
expiresAt: string;
|
|
}
|
|
|
|
interface ObjectStorageMultipartUploadCreateResult {
|
|
bucket: string;
|
|
key: string;
|
|
uploadId: string;
|
|
expiresAt: string;
|
|
}
|
|
|
|
interface ObjectStorageMultipartUploadPartURLResult {
|
|
bucket: string;
|
|
key: string;
|
|
uploadId: string;
|
|
partNumber: number;
|
|
method: "PUT";
|
|
url: string;
|
|
headers?: Record<string, string>;
|
|
expiresAt: string;
|
|
}
|
|
|
|
interface ObjectStorageMultipartUploadPart {
|
|
partNumber: number;
|
|
eTag: string;
|
|
}
|
|
|
|
async function uploadWithFormData<T>(path: string, formData: FormData) {
|
|
const response = await apiFetch(
|
|
path,
|
|
{
|
|
method: "POST",
|
|
body: formData,
|
|
},
|
|
{ includeJSONContentType: false },
|
|
);
|
|
if (!response.ok) {
|
|
const text = await response.text();
|
|
throw new Error(text || `Request failed: ${response.status}`);
|
|
}
|
|
return (await response.json()) as T;
|
|
}
|
|
|
|
async function uploadMultipartObject(
|
|
file: File,
|
|
options: { connectionId: string; bucket?: string; key: string; contentType?: string },
|
|
onProgress?: (progress: ObjectStorageUploadProgress) => void,
|
|
) {
|
|
const initiated = await apiRequest<ObjectStorageMultipartUploadCreateResult>(
|
|
`/api/object-storage/uploads/multipart/initiate${buildObjectStorageParams({ connectionId: options.connectionId })}`,
|
|
{ method: "POST" },
|
|
{
|
|
bucket: options.bucket,
|
|
key: options.key,
|
|
contentType: options.contentType,
|
|
expiresInSeconds: PRESIGN_EXPIRES_SECONDS,
|
|
},
|
|
);
|
|
const parts = objectStorageMultipartParts(file.size, MULTIPART_PART_BYTES);
|
|
const uploadedParts: ObjectStorageMultipartUploadPart[] = [];
|
|
let completedBytes = 0;
|
|
let nextIndex = 0;
|
|
let aborted = false;
|
|
const activePartBytes = new Map<number, number>();
|
|
|
|
const reportMultipartProgress = (partNumber?: number) => {
|
|
const activeBytes = Array.from(activePartBytes.values()).reduce((total, loadedBytes) => total + loadedBytes, 0);
|
|
onProgress?.({
|
|
mode: "multipart",
|
|
loadedBytes: Math.min(completedBytes + activeBytes, file.size),
|
|
totalBytes: file.size,
|
|
partNumber,
|
|
partCount: parts.length,
|
|
});
|
|
};
|
|
|
|
const abortUpload = async () => {
|
|
if (aborted) {
|
|
return;
|
|
}
|
|
aborted = true;
|
|
try {
|
|
await apiRequest(
|
|
`/api/object-storage/uploads/multipart/abort${buildObjectStorageParams({ connectionId: options.connectionId })}`,
|
|
{ method: "POST" },
|
|
{ bucket: options.bucket, key: options.key, uploadId: initiated.uploadId },
|
|
);
|
|
} catch {
|
|
// Preserve the original upload failure.
|
|
}
|
|
};
|
|
|
|
const uploadNext = async () => {
|
|
while (nextIndex < parts.length) {
|
|
const part = parts[nextIndex];
|
|
nextIndex += 1;
|
|
const partURL = await apiRequest<ObjectStorageMultipartUploadPartURLResult>(
|
|
`/api/object-storage/uploads/multipart/part-url${buildObjectStorageParams({ connectionId: options.connectionId })}`,
|
|
{ method: "POST" },
|
|
{
|
|
bucket: options.bucket,
|
|
key: options.key,
|
|
uploadId: initiated.uploadId,
|
|
partNumber: part.partNumber,
|
|
expiresInSeconds: PRESIGN_EXPIRES_SECONDS,
|
|
},
|
|
);
|
|
const partSize = part.end - part.start;
|
|
activePartBytes.set(part.partNumber, 0);
|
|
const response = await uploadObjectStoragePresignedURL(
|
|
partURL.url,
|
|
file.slice(part.start, part.end),
|
|
partURL.headers,
|
|
(loadedBytes) => {
|
|
activePartBytes.set(part.partNumber, Math.min(loadedBytes, partSize));
|
|
reportMultipartProgress(part.partNumber);
|
|
},
|
|
);
|
|
const eTag = parseObjectStorageUploadETag(response.headers.get("ETag"));
|
|
if (!eTag) {
|
|
throw new Error(`Object storage upload part ${part.partNumber} did not return ETag`);
|
|
}
|
|
uploadedParts.push({ partNumber: part.partNumber, eTag });
|
|
activePartBytes.delete(part.partNumber);
|
|
completedBytes += partSize;
|
|
reportMultipartProgress(part.partNumber);
|
|
}
|
|
};
|
|
|
|
try {
|
|
onProgress?.({ mode: "multipart", loadedBytes: 0, totalBytes: file.size, partCount: parts.length });
|
|
await Promise.all(Array.from({ length: Math.min(MULTIPART_CONCURRENCY, parts.length) }, () => uploadNext()));
|
|
uploadedParts.sort((left, right) => left.partNumber - right.partNumber);
|
|
return await apiRequest<ObjectStorageObjectMutationResult>(
|
|
`/api/object-storage/uploads/multipart/complete${buildObjectStorageParams({ connectionId: options.connectionId })}`,
|
|
{ method: "POST" },
|
|
{ bucket: options.bucket, key: options.key, uploadId: initiated.uploadId, parts: uploadedParts },
|
|
);
|
|
} catch (error) {
|
|
await abortUpload();
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function uploadDirectObject(
|
|
file: File,
|
|
options: { connectionId: string; bucket?: string; key: string; contentType?: string },
|
|
onProgress?: (progress: ObjectStorageUploadProgress) => void,
|
|
) {
|
|
const presigned = await apiRequest<ObjectStoragePresignedUploadResult>(
|
|
`/api/object-storage/uploads/presign-put${buildObjectStorageParams({ connectionId: options.connectionId })}`,
|
|
{ method: "POST" },
|
|
{
|
|
bucket: options.bucket,
|
|
key: options.key,
|
|
contentType: options.contentType,
|
|
expiresInSeconds: PRESIGN_EXPIRES_SECONDS,
|
|
},
|
|
);
|
|
onProgress?.({ mode: "direct", loadedBytes: 0, totalBytes: file.size });
|
|
await uploadObjectStoragePresignedURL(presigned.url, file, presigned.headers, (loadedBytes) => {
|
|
onProgress?.({
|
|
mode: "direct",
|
|
loadedBytes: Math.min(loadedBytes, file.size),
|
|
totalBytes: file.size,
|
|
});
|
|
});
|
|
onProgress?.({ mode: "direct", loadedBytes: file.size, totalBytes: file.size });
|
|
const prefix = options.key.includes("/") ? options.key.slice(0, options.key.lastIndexOf("/")) : "";
|
|
const listed = await apiRequest<ObjectStorageBrowseResult>(
|
|
`/api/object-storage/objects${buildObjectStorageParams({
|
|
connectionId: options.connectionId,
|
|
bucket: options.bucket,
|
|
prefix,
|
|
})}`,
|
|
);
|
|
const object = listed.items.find((item) => item.key === options.key);
|
|
if (!object) {
|
|
throw new Error("Object storage upload completed but uploaded object was not found");
|
|
}
|
|
return { object };
|
|
}
|
|
|
|
export const objectStorageApi = {
|
|
getOverview: (options?: { connectionId?: string }) =>
|
|
apiRequest<ObjectStorageOverview>(`/api/object-storage${buildObjectStorageParams(options)}`),
|
|
listObjects: (options: { connectionId: string; bucket?: string; prefix?: string; marker?: string; limit?: number }) =>
|
|
apiRequest<ObjectStorageBrowseResult>(`/api/object-storage/objects${buildObjectStorageParams(options)}`),
|
|
putObject: (
|
|
payload: { bucket?: string; key: string; content: string; contentType?: string },
|
|
options: { connectionId: string },
|
|
) =>
|
|
apiRequest<ObjectStorageObjectMutationResult>(
|
|
`/api/object-storage/objects${buildObjectStorageParams(options)}`,
|
|
{ method: "POST" },
|
|
payload,
|
|
),
|
|
uploadObject: async (
|
|
file: File,
|
|
options: { connectionId: string; bucket?: string; key: string; contentType?: string },
|
|
onProgress?: (progress: ObjectStorageUploadProgress) => void,
|
|
) => {
|
|
if (objectStorageUploadMode(file.size) === "multipart") {
|
|
return uploadMultipartObject(file, options, onProgress);
|
|
}
|
|
return uploadDirectObject(file, options, onProgress);
|
|
},
|
|
prepareBatchUpload: (
|
|
payload: {
|
|
bucket?: string;
|
|
prefix?: string;
|
|
conflictPolicy: ObjectStorageBatchConflictPolicy;
|
|
items: ObjectStorageBatchUploadItemInput[];
|
|
},
|
|
options: { connectionId: string },
|
|
) =>
|
|
apiRequest<ObjectStorageBatchUploadPrepareResult>(
|
|
`/api/object-storage/uploads/batch/prepare${buildObjectStorageParams(options)}`,
|
|
{ method: "POST" },
|
|
payload,
|
|
),
|
|
uploadObjectWithServerFallback: async (
|
|
file: File,
|
|
options: { connectionId: string; bucket?: string; key: string; contentType?: string },
|
|
) => {
|
|
const formData = new FormData();
|
|
formData.append("file", file);
|
|
return uploadWithFormData<ObjectStorageObjectMutationResult>(
|
|
`/api/object-storage/objects/upload${buildObjectStorageParams(options)}`,
|
|
formData,
|
|
);
|
|
},
|
|
renameObject: (
|
|
payload: { bucket?: string; key: string; newKey: string; recursive?: boolean },
|
|
options: { connectionId: string },
|
|
) =>
|
|
apiRequest<ObjectStorageObjectMutationResult>(
|
|
`/api/object-storage/objects${buildObjectStorageParams(options)}`,
|
|
{ method: "PATCH" },
|
|
payload,
|
|
),
|
|
moveObjects: (
|
|
payload: {
|
|
bucket?: string;
|
|
destinationPrefix: string;
|
|
items: Array<{ key: string; kind?: string; recursive?: boolean }>;
|
|
},
|
|
options: { connectionId: string },
|
|
) =>
|
|
apiRequest<ObjectStorageObjectMutationResult>(
|
|
`/api/object-storage/objects${buildObjectStorageParams(options)}`,
|
|
{ method: "PATCH" },
|
|
payload,
|
|
),
|
|
deleteObject: (options: { connectionId: string; bucket?: string; key: string; recursive?: boolean }) =>
|
|
apiRequest<ObjectStorageObjectDeleteResult>(`/api/object-storage/objects${buildObjectStorageParams(options)}`, {
|
|
method: "DELETE",
|
|
}),
|
|
restoreObject: (
|
|
payload: { bucket?: string; trashKey: string; originalKey: string; recursive?: boolean },
|
|
options: { connectionId: string },
|
|
) =>
|
|
apiRequest<ObjectStorageObjectMutationResult>(
|
|
`/api/object-storage/objects/restore${buildObjectStorageParams(options)}`,
|
|
{ method: "POST" },
|
|
payload,
|
|
),
|
|
buildContentURL: (options: { connectionId: string; bucket?: string; key: string; transcode?: string }) =>
|
|
buildAPIURL(`/api/object-storage/content${buildObjectStorageParams(options)}`),
|
|
readObjectTextPreview: async (options: {
|
|
connectionId: string;
|
|
bucket?: string;
|
|
key: string;
|
|
}): Promise<ObjectStorageTextPreviewResult> => {
|
|
const response = await apiFetch(
|
|
`/api/object-storage/content${buildObjectStorageParams({
|
|
...options,
|
|
preview: true,
|
|
})}`,
|
|
undefined,
|
|
{ includeJSONContentType: false },
|
|
);
|
|
if (!response.ok) {
|
|
const text = await response.text();
|
|
throw new Error(text || `Request failed: ${response.status}`);
|
|
}
|
|
const content = await response.text();
|
|
const parseBytesHeader = (headerName: string) => {
|
|
return parseObjectStoragePreviewBytesHeader(response.headers.get(headerName));
|
|
};
|
|
return {
|
|
content,
|
|
truncated: response.headers.get(OBJECT_STORAGE_PREVIEW_TRUNCATED_HEADER) === "true",
|
|
originalSizeBytes: parseBytesHeader(OBJECT_STORAGE_PREVIEW_ORIGINAL_SIZE_HEADER),
|
|
returnedSizeBytes: parseBytesHeader(OBJECT_STORAGE_PREVIEW_RETURNED_SIZE_HEADER),
|
|
};
|
|
},
|
|
};
|