Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
"@apify/actor-templates": "^0.1.5",
"@apify/consts": "^2.36.0",
"@apify/input_schema": "^3.17.0",
"@apify/json_schemas": "^0.13.0",
"@apify/utilities": "^2.18.0",
"@crawlee/memory-storage": "^3.12.0",
"@inquirer/core": "^11.0.0",
Expand Down
4 changes: 2 additions & 2 deletions src/commands/_register.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import { RunsIndexCommand } from './runs/_index.js';
import { SecretsIndexCommand } from './secrets/_index.js';
import { TasksIndexCommand } from './task/_index.js';
import { TelemetryIndexCommand } from './telemetry/_index.js';
import { ValidateInputSchemaCommand } from './validate-schema.js';
import { ValidateSchemaCommand } from './validate-schema.js';

export const apifyCommands = [
// namespaces
Expand Down Expand Up @@ -62,7 +62,7 @@ export const apifyCommands = [
TopLevelPullCommand,
ToplevelPushCommand,
RunCommand,
ValidateInputSchemaCommand,
ValidateSchemaCommand,
HelpCommand,

// test commands
Expand Down
18 changes: 13 additions & 5 deletions src/commands/edit-input-schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,19 @@ export class EditInputSchemaCommand extends ApifyCommand<typeof EditInputSchemaC
static override aliases = ['eis'];

async run() {
// This call fails if no input schema is found on any of the default locations
const { inputSchema: existingSchema, inputSchemaPath } = await readInputSchema({
forcePath: this.args.path,
cwd: process.cwd(),
});
let result: Awaited<ReturnType<typeof readInputSchema>>;

try {
result = await readInputSchema({
forcePath: this.args.path,
cwd: process.cwd(),
});
} catch (err) {
error({ message: (err as Error).message });
return;
}

const { inputSchema: existingSchema, inputSchemaPath } = result;

if (existingSchema && !inputSchemaPath) {
// If path is not returned, it means the input schema must be directly embedded as object in actor.json
Expand Down
9 changes: 8 additions & 1 deletion src/commands/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,14 @@ export class RunCommand extends ApifyCommand<typeof RunCommand> {
* @param inputOverride Optional input received through command flags
*/
private async validateAndStoreInput(inputOverride?: { input: Record<string, unknown>; source: string }) {
const { inputSchema } = await readInputSchema({ cwd: process.cwd() });
let inputSchema: Record<string, unknown> | null | undefined;

try {
({ inputSchema } = await readInputSchema({ cwd: process.cwd() }));
} catch (err) {
warning({ message: (err as Error).message });
inputSchema = null;
}

if (!inputSchema) {
if (!inputOverride) {
Expand Down
138 changes: 122 additions & 16 deletions src/commands/validate-schema.ts
Original file line number Diff line number Diff line change
@@ -1,41 +1,147 @@
import process from 'node:process';

import { validateInputSchema } from '@apify/input_schema';

import { ApifyCommand } from '../lib/command-framework/apify-command.js';
import { Args } from '../lib/command-framework/args.js';
import { LOCAL_CONFIG_PATH } from '../lib/consts.js';
import { readAndValidateInputSchema } from '../lib/input_schema.js';
import { success } from '../lib/outputs.js';
import { CommandExitCodes, LOCAL_CONFIG_PATH } from '../lib/consts.js';
import {
readAndValidateInputSchema,
readInputSchema,
readStorageSchema,
validateDatasetSchema,
validateKvsSchema,
validateOutputSchema,
} from '../lib/input_schema.js';
import { error, info, success } from '../lib/outputs.js';
import { Ajv2019 } from '../lib/utils.js';

export class ValidateInputSchemaCommand extends ApifyCommand<typeof ValidateInputSchemaCommand> {
export class ValidateSchemaCommand extends ApifyCommand<typeof ValidateSchemaCommand> {
static override name = 'validate-schema' as const;

static override description = `Validates Actor input schema from one of these locations (in priority order):
1. Object in '${LOCAL_CONFIG_PATH}' under "input" key
2. JSON file path in '${LOCAL_CONFIG_PATH}' "input" key
3. .actor/INPUT_SCHEMA.json
4. INPUT_SCHEMA.json
static override description = `Validates Actor schemas.

When a path argument is provided, validates only the input schema at that path.

Optionally specify custom schema path to validate.`;
When no path is provided, validates all schemas found in '${LOCAL_CONFIG_PATH}':
- Input schema (from "input" key or default locations)
- Dataset schema (from "storages.dataset")
- Output schema (from "output")
- Key-Value Store schema (from "storages.keyValueStore")`;

static override args = {
path: Args.string({
required: false,
description: 'Optional path to your INPUT_SCHEMA.json file. If not provided ./INPUT_SCHEMA.json is used.',
description:
'Optional path to your INPUT_SCHEMA.json file. If not provided, validates all schemas in actor.json.',
}),
};

static override hiddenAliases = ['vis'];

async run() {
if (this.args.path) {
await this.validateInputSchemaAtPath(this.args.path);
return;
}

await this.validateAllSchemas();
}

private async validateInputSchemaAtPath(forcePath: string) {
await readAndValidateInputSchema({
forcePath: this.args.path,
forcePath,
cwd: process.cwd(),
getMessage: (path) =>
path
? `Validating input schema at ${path}`
: `Validating input schema embedded in '${LOCAL_CONFIG_PATH}'`,
getMessage: (path) => `Validating input schema at ${path ?? forcePath}`,
});

success({ message: 'Input schema is valid.' });
}

private async validateAllSchemas() {
const cwd = process.cwd();
let foundAny = false;
let hasErrors = false;

// Input schema — not using readAndValidateInputSchema here because it throws
// when no schema is found; in the all-schemas scan, a missing input schema
// should be silently skipped, not treated as an error.
try {
const { inputSchema, inputSchemaPath } = await readInputSchema({ cwd, throwOnMissing: true });

if (inputSchema) {
foundAny = true;

const location = inputSchemaPath ? `at ${inputSchemaPath}` : `embedded in '${LOCAL_CONFIG_PATH}'`;
info({ message: `Validating input schema ${location}` });

const validator = new Ajv2019({ strict: false });
validateInputSchema(validator, inputSchema);
success({ message: 'Input schema is valid.' });
}
} catch (err) {
foundAny = true;
hasErrors = true;
error({ message: (err as Error).message });
}

// Storage schemas (Dataset, Output, Key-Value Store)
const storageSchemas = [
{
label: 'Dataset',
read: () => readStorageSchema({ cwd, key: 'dataset', label: 'Dataset', throwOnMissing: true }),
validate: validateDatasetSchema,
},
{
label: 'Output',
read: () =>
readStorageSchema({
cwd,
key: 'output',
label: 'Output',
getRef: (config) => config?.output,
throwOnMissing: true,
}),
validate: validateOutputSchema,
},
{
label: 'Key-Value Store',
read: () =>
readStorageSchema({ cwd, key: 'keyValueStore', label: 'Key-Value Store', throwOnMissing: true }),
validate: validateKvsSchema,
},
];

for (const { label, read, validate } of storageSchemas) {
try {
const result = read();

if (result) {
foundAny = true;

const location = result.schemaPath
? `at ${result.schemaPath}`
: `embedded in '${LOCAL_CONFIG_PATH}'`;
info({ message: `Validating ${label} schema ${location}` });

validate(result.schema);
success({ message: `${label} schema is valid.` });
}
} catch (err) {
foundAny = true;
hasErrors = true;
error({ message: (err as Error).message });
}
}

if (!foundAny) {
throw new Error(
`No schemas found. Make sure '${LOCAL_CONFIG_PATH}' exists and defines at least one schema.`,
);
}

if (hasErrors) {
process.exitCode = CommandExitCodes.InvalidInput;
}
}
}
79 changes: 75 additions & 4 deletions src/lib/input_schema.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
import { existsSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';

import type { Ajv, ErrorObject } from 'ajv';
import { cloneDeep } from 'es-toolkit';

import { KEY_VALUE_STORE_KEYS } from '@apify/consts';
import { validateInputSchema } from '@apify/input_schema';
import {
getDatasetSchemaValidator,
getKeyValueStoreSchemaValidator,
getOutputSchemaValidator,
} from '@apify/json_schemas';

import { ACTOR_SPECIFICATION_FOLDER, LOCAL_CONFIG_PATH } from './consts.js';
import { info, warning } from './outputs.js';
Expand All @@ -24,7 +30,15 @@ const DEFAULT_INPUT_SCHEMA_PATHS = [
* In such a case, path would be set to the location
* where the input schema would be expected to be found (and e.g. can be created there).
*/
export const readInputSchema = async ({ forcePath, cwd }: { forcePath?: string; cwd: string }) => {
export const readInputSchema = async ({
forcePath,
cwd,
throwOnMissing = false,
}: {
forcePath?: string;
cwd: string;
throwOnMissing?: boolean;
}) => {
if (forcePath) {
return {
inputSchema: getJsonFileContent(forcePath),
Expand All @@ -34,7 +48,7 @@ export const readInputSchema = async ({ forcePath, cwd }: { forcePath?: string;

const localConfig = getLocalConfig(cwd);

if (typeof localConfig?.input === 'object') {
if (typeof localConfig?.input === 'object' && localConfig.input !== null) {
return {
inputSchema: localConfig.input as Record<string, unknown>,
inputSchemaPath: null,
Expand All @@ -43,8 +57,25 @@ export const readInputSchema = async ({ forcePath, cwd }: { forcePath?: string;

if (typeof localConfig?.input === 'string') {
const fullPath = join(cwd, ACTOR_SPECIFICATION_FOLDER, localConfig.input);
const schema = getJsonFileContent(fullPath);

if (!schema) {
if (throwOnMissing) {
throw new Error(`Input schema file not found at ${fullPath} (referenced in '${LOCAL_CONFIG_PATH}').`);
}

warning({
message: `Input schema file not found at ${fullPath} (referenced in '${LOCAL_CONFIG_PATH}').`,
});

return {
inputSchema: null,
inputSchemaPath: fullPath,
};
}

return {
inputSchema: getJsonFileContent(fullPath),
inputSchema: schema,
inputSchemaPath: fullPath,
};
}
Expand Down Expand Up @@ -115,11 +146,13 @@ export const readStorageSchema = ({
key,
label,
getRef,
throwOnMissing = false,
}: {
cwd: string;
key: string;
label: string;
getRef?: (config: ReturnType<typeof getLocalConfig>) => unknown;
throwOnMissing?: boolean;
}): { schema: Record<string, unknown>; schemaPath: string | null } | null => {
const localConfig = getLocalConfig(cwd);

Expand All @@ -137,6 +170,12 @@ export const readStorageSchema = ({
const schema = getJsonFileContent(fullPath);

if (!schema) {
if (throwOnMissing) {
throw new Error(
`${label} schema file not found at ${fullPath} (referenced in '${LOCAL_CONFIG_PATH}').`,
);
}

warning({
message: `${label} schema file not found at ${fullPath} (referenced in '${LOCAL_CONFIG_PATH}').`,
});
Expand Down Expand Up @@ -255,8 +294,40 @@ export const getDefaultsFromInputSchema = (inputSchema: any) => {
return defaults;
};

function formatSchemaValidationErrors(errors: ErrorObject[], schemaName: string): string {
const details = errors
.map((err) => {
const path = err.instancePath ? ` at ${err.instancePath}` : '';
return ` - ${err.message}${path}`;
})
.join('\n');

return `${schemaName} schema is not valid:\n${details}`;
}

export function validateDatasetSchema(schema: Record<string, unknown>): void {
const validate = getDatasetSchemaValidator();
if (!validate(schema)) {
throw new Error(formatSchemaValidationErrors(validate.errors!, 'Dataset'));
}
}

export function validateOutputSchema(schema: Record<string, unknown>): void {
const validate = getOutputSchemaValidator();
if (!validate(schema)) {
throw new Error(formatSchemaValidationErrors(validate.errors!, 'Output'));
}
}

export function validateKvsSchema(schema: Record<string, unknown>): void {
const validate = getKeyValueStoreSchemaValidator();
if (!validate(schema)) {
throw new Error(formatSchemaValidationErrors(validate.errors!, 'Key-Value Store'));
}
}

// Lots of code copied from @apify-packages/actor, this really should be moved to the shared input_schema package
export const getAjvValidator = (inputSchema: any, ajvInstance: import('ajv').Ajv) => {
export const getAjvValidator = (inputSchema: any, ajvInstance: Ajv) => {
const copyOfSchema = cloneDeep(inputSchema);
copyOfSchema.required = [];

Expand Down
Loading