schema validation

This commit is contained in:
Merith 2025-06-22 14:39:53 -07:00
parent 0eb645d264
commit 8884c53f57
7 changed files with 445 additions and 27 deletions

1
go.mod
View file

@ -4,6 +4,7 @@ go 1.24.3
require (
github.com/magefile/mage v1.15.0
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
github.com/xeipuuv/gojsonschema v1.2.0
)

2
go.sum
View file

@ -4,6 +4,8 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

View file

@ -4,22 +4,36 @@
package main
import (
"archive/zip"
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"unicode"
"strings"
"unicode/utf8"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/xeipuuv/gojsonschema"
"github.com/santhosh-tekuri/jsonschema/v5"
)
const (
zipName = "DigitalStorageTweaks.zip"
contentDir = "./ContentLib"
pluginFile = "./DigitalStorageTweaks.uplugin"
schemaPath = "./schema/CL_Recipe.json" // Local schema path
)
var (
// Binary file extensions to skip line ending conversion
binaryExtensions = map[string]bool{
".png": true, ".jpg": true, ".jpeg": true, ".bmp": true, ".gif": true,
".dds": true, ".tga": true, ".psd": true, ".fbx": true, ".uasset": true,
".umap": true,
}
)
// Default target
@ -44,57 +58,125 @@ func Package() error {
}
}
// Create zip
if err := sh.Run("7z", "a", "-r", zipName, "Windows/", "LinuxServer/", "WindowsServer/"); err != nil {
// Create zip file
if err := createZip(zipName, "Windows", "WindowsServer", "LinuxServer"); err != nil {
return fmt.Errorf("creating zip: %w", err)
}
// Clean temp dirs
return Clean("Windows", "WindowsServer", "LinuxServer")
}
// Validate checks all content files
func Validate() error {
fmt.Println("Validating files...")
// Check plugin file
if err := checkFile(pluginFile); err != nil {
const (
baseSchemaURL = "https://raw.githubusercontent.com/budak7273/ContentLib_Documentation/main/JsonSchemas/"
localSchemaDir = "./schema"
)
compiler := jsonschema.NewCompiler()
// Disable remote fetching
compiler.LoadURL = func(url string) (io.ReadCloser, error) {
return nil, fmt.Errorf("remote schema loading disabled: %s", url)
}
// Walk through the schema directory and add all .json files
err := filepath.Walk(localSchemaDir, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
if filepath.Ext(path) != ".json" {
return nil
}
relPath, err := filepath.Rel(localSchemaDir, path)
if err != nil {
return fmt.Errorf("resolving relative path: %w", err)
}
// Build the full schema ID
schemaID := baseSchemaURL + filepath.ToSlash(relPath)
data, err := ioutil.ReadFile(path)
if err != nil {
return fmt.Errorf("reading %s: %w", path, err)
}
err = compiler.AddResource(schemaID, bytes.NewReader(data))
if err != nil {
return fmt.Errorf("adding schema %s: %w", schemaID, err)
}
return nil
})
if err != nil {
return fmt.Errorf("failed to load schemas: %w", err)
}
// Main schema (must match its $id)
mainSchemaID := baseSchemaURL + "CL_Recipe.json"
// Compile the main schema
schema, err := compiler.Compile(mainSchemaID)
if err != nil {
return fmt.Errorf("invalid main schema: %w", err)
}
// Validate plugin file
if err := validateFile(pluginFile, schema); err != nil {
return fmt.Errorf("plugin file: %w", err)
}
// Check content directory
// Validate all JSON files in the content directory
return filepath.Walk(contentDir, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
return checkFile(path)
return validateFile(path, schema)
})
}
func checkFile(path string) error {
func validateFile(path string, schema *jsonschema.Schema) error {
// Skip binary files
switch filepath.Ext(path) {
case ".png", ".jpg", ".jpeg", ".bmp", ".gif", ".dds", ".tga", ".psd", ".fbx", ".uasset", ".umap":
ext := filepath.Ext(path)
if binaryExtensions[ext] {
return nil
case ".json":
return validateJSON(path)
default:
return validateEncoding(path)
}
if ext == ".json" {
return validateJSON(path, schema)
}
return validateEncoding(path)
}
func validateJSON(path string) error {
schemaLoader := gojsonschema.NewReferenceLoader("https://raw.githubusercontent.com/budak7273/ContentLib_Documentation/main/JsonSchemas/CL_Recipe.json")
documentLoader := gojsonschema.NewReferenceLoader("file:///" + filepath.ToSlash(path))
result, err := gojsonschema.Validate(schemaLoader, documentLoader)
func validateJSON(path string, schema *jsonschema.Schema) error {
content, err := ioutil.ReadFile(path)
if err != nil {
return fmt.Errorf("schema validation failed: %w", err)
return fmt.Errorf("failed to read JSON file: %w", err)
}
if !result.Valid() {
return fmt.Errorf("invalid JSON schema: %v", result.Errors())
// Remove comment lines (starting with //)
var filteredContent []byte
for _, line := range bytes.Split(content, []byte("\n")) {
trimmed := bytes.TrimSpace(line)
if !bytes.HasPrefix(trimmed, []byte("//")) && len(trimmed) > 0 {
filteredContent = append(filteredContent, line...)
filteredContent = append(filteredContent, '\n')
}
}
// Validate JSON
var v interface{}
if err := json.Unmarshal(filteredContent, &v); err != nil {
return fmt.Errorf("invalid JSON in %s: %w", path, err)
}
if err := schema.Validate(v); err != nil {
return fmt.Errorf("schema validation failed for %s: %w", path, err)
}
return nil
}
@ -105,11 +187,16 @@ func validateEncoding(path string) error {
}
// Check for non-ASCII
for i := 0; i < len(content); i++ {
if content[i] > unicode.MaxASCII {
for i := 0; i < len(content); {
b := content[i]
if b > 127 { // Non-ASCII
r, _ := utf8.DecodeRune(content[i:])
if r == utf8.RuneError {
return fmt.Errorf("invalid UTF-8 sequence at position %d", i)
}
return fmt.Errorf("non-ASCII character %U at position %d", r, i)
}
i++
}
return nil
}
@ -124,3 +211,101 @@ func Clean(list ...string) error {
}
return nil
}
// createZip creates a zip file from the specified directories
func createZip(zipPath string, dirs ...string) error {
// Create zip file
zipFile, err := os.Create(zipPath)
if err != nil {
return fmt.Errorf("creating zip file: %w", err)
}
defer zipFile.Close()
// Create zip writer
zipWriter := zip.NewWriter(zipFile)
defer zipWriter.Close()
// Add each directory to the zip
for _, dir := range dirs {
if err := addDirToZip(zipWriter, dir); err != nil {
return fmt.Errorf("adding %s to zip: %w", dir, err)
}
}
return nil
}
// addDirToZip adds a directory to the zip, converting text files to DOS line endings
func addDirToZip(zipWriter *zip.Writer, dirPath string) error {
return filepath.Walk(dirPath, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Create relative path for the zip file
relPath, err := filepath.Rel(dirPath, filePath)
if err != nil {
return err
}
zipPath := filepath.Join(filepath.Base(dirPath), relPath)
// Create zip file header
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
header.Name = filepath.ToSlash(zipPath) // Use forward slashes for zip compatibility
// Use compression for all files
header.Method = zip.Deflate
// Handle directories
if info.IsDir() {
header.Name += "/"
_, err := zipWriter.CreateHeader(header)
return err
}
// Open source file
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
// Create writer in zip
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return err
}
// Process based on file type
ext := strings.ToLower(filepath.Ext(filePath))
if binaryExtensions[ext] {
// Binary file - copy directly
_, err = io.Copy(writer, file)
return err
} else {
// Text file - read content and convert line endings
content, err := ioutil.ReadAll(file)
if err != nil {
return err
}
// Convert LF to CRLF
content = convertToDOSLineEndings(content)
// Write to zip
_, err = writer.Write(content)
return err
}
})
}
// convertToDOSLineEndings converts LF to CRLF while preserving existing CRLF
func convertToDOSLineEndings(content []byte) []byte {
// First normalize to LF
normalized := bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n"))
// Then convert to CRLF
return bytes.ReplaceAll(normalized, []byte("\n"), []byte("\r\n"))
}

View file

@ -0,0 +1,9 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://raw.githubusercontent.com/budak7273/ContentLib_Documentation/main/JsonSchemas/CL_NameResolvedAsset.json",
"title": "ContentLib Name Resolved Asset",
"description": "An asset that can be specified either with a full path, or a shortened name for ContentLib to resolve. See more info here: https://docs-dev.ficsit.app/contentlib/latest/BackgroundInfo/AutomaticNameResolving.html",
"$comment": "This file does not represent ContentLib asset type! It's a resource used by other JSON Schemas. Please read https://docs-dev.ficsit.app/contentlib/latest/BackgroundInfo/AutomaticNameResolving.html",
"type": "string",
"minLength": 1
}

189
schema/CL_Recipe.json Normal file
View file

@ -0,0 +1,189 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://raw.githubusercontent.com/budak7273/ContentLib_Documentation/main/JsonSchemas/CL_Recipe.json",
"title": "ContentLib Recipe",
"description": "ContentLib Recipe struct definitions can be made from this JSON, and when converted to JSON, will follow this form.",
"$comment": "This is NOT a copy-paste template! Check the docs to see how to use this! https://docs.ficsit.app/contentlib/latest/index.html",
"type": "object",
"additionalProperties": false,
"properties": {
"Name": {
"type": "string",
"description": "Override for the recipe's name, which is displayed inside the manual crafting bench, build gun, or any machines you set it to be used in. By default, recipes take the name of their first product item."
},
"Ingredients": {
"type": "array",
"description": "ItemAmounts which form the ingredients. Be sure to not exceed an item's stack size, and remember that 1000 fluid units displays as 1 Liter ingame. See 'ClearIngredients' to append instead of replace.",
"items": {
"$ref": "./Components/CL_ItemAmount.json"
},
"examples": [
[
{
"Item": "Desc_CopperIngot",
"Amount": 1
},
{
"Item": "Desc_IronIngot",
"Amount": 1
}
]
]
},
"Products": {
"type": "array",
"description": "ItemAmounts which form the recipe outputs. Be sure to not exceed an item's stack size, and remember that 1000 fluid units displays as 1 Liter ingame. Manual recipes should only have 1 product. See 'ClearProducts' to append instead of replace.",
"items": {
"$ref": "./Components/CL_ItemAmount.json"
},
"examples": [
[
{
"Item": "Desc_Biofuel",
"Amount": 10
}
]
]
},
"ProducedIn": {
"type": "array",
"description": "Automatic name resolved asset. Machines/builders which this recipe can be used in. The examples are not an exhaustive list. Use 'manual' for all manual workbenches; this is a workaround and will be fixed in the future. See 'ClearBuilders' to append instead of replace.",
"uniqueItems": true,
"items": {
"$ref": "./Components/CL_NameResolvedAsset.json",
"examples": [
"manual",
"Build_SmelterMk1",
"Build_ConstructorMk1",
"Build_AssemblerMk1",
"Build_FoundryMk1",
"Build_ManufacturerMk1",
"Build_OilRefinery",
"Build_Packager",
"Build_Blender",
"Build_HadronCollider",
"Build_Converter",
"Build_QuantumEncoder",
"BP_BuildGun"
],
"allOf": [
{
"$comment": "The `manual` workaround should be used instead of these (see the docs https://docs.ficsit.app/contentlib/latest/Tutorials/CreateRecipe.html).",
"if": {
"anyOf": [
{
"const": "FGWorkBench"
},
{
"const": "Build_Workshop"
},
{
"const": "Build_WorkBench"
},
{
"const": "Build_WorkBenchIntegrated"
}
]
},
"then": {
"const": "manual"
}
}
]
}
},
"OverrideCategory": {
"type": "string",
"description": "By default, recipes take on the category of their first product item. Specifying something here causes it to be used as the category instead. Either a base-game category, or a string which ContentLib will automatically turn into a category for you.",
"examples": [
"Cat_Other",
"MyCustomAutoCreatedCategory"
]
},
"ManufacturingDuration": {
"type": "number",
"description": "The time it takes for a machine to process this recipe in seconds. Recipe energy consumption is usually this times the machine's power usage."
},
"ManualManufacturingMultiplier": {
"type": "number",
"description": "If you want to use the same recipe for machines and the craft bench, this defines how much longer the crafting of this recipe should take in the craft bench. With the multiplier set to the default of 0, the number of clicks to craft is the number of seconds it takes to craft divided by two.",
"examples": [
0.0
]
},
"UnlockedBy": {
"type": "array",
"description": "Automatic name resolved asset. Schematics to add this recipe to. Alternative way of tying recipes to schematics; the standard way is to use the schematic's 'Recipes' field. This is useful if you don't want to also make a patch to add your recipe to a base-game schematic. Something must unlock the recipe, else it will never be accessible in-game. If you want it to be unlocked by default, use Schematic_StartingRecipes.",
"items": {
"$ref": "./Components/CL_NameResolvedAsset.json"
},
"examples": [
[
"Schematic_StartingRecipes"
],
[
"Schematic_1-1"
]
]
},
"VariablePowerConsumptionFactor": {
"type": "number",
"description": "Only works in variable consumption machines (the Particle Accelerator in base game). Added to the VariablePowerConsumptionFactor to get the 'maximum' power consumption the recipe can scale up to over its cycle."
},
"VariablePowerConsumptionConstant": {
"type": "number",
"description": "Only works in variable consumption machines (the Particle Accelerator in base game). The 'minimum' power consumption the recipe can scale down to over its cycle."
},
"ManufacturingMenuPriority": {
"type": "number",
"description": "Controls the order that recipes appear within their category. Lower values means earlier in the menu. Negatives and decimals allowed."
},
"OverrideName": {
"type": "boolean",
"description": "For patches, should the Recipe this is used on have its 'use override name' flag enabled? Default is true when 'Name' exists, false when not. This can be used to ex. revert a recipe's displayed name to that of its first product, instead of the custom name that was specified for it."
},
"ClearIngredients": {
"type": "boolean",
"description": "For patches, should the Recipe this is used on have its existing Ingredients array cleared before adding to it? Default is true when 'Ingredients' exists, false when not."
},
"ClearProducts": {
"type": "boolean",
"description": "For patches, should the Recipe this is used on have its existing Products array cleared before adding to it? Default is true when 'Products' exists, false when not."
},
"ClearBuilders": {
"type": "boolean",
"description": "For patches, should the Recipe this is used on have its existing ProducedIn array cleared before adding to it? Default is true when 'ProducedIn' exists, false when not."
}
},
"allOf": [
{
"$comment": "Manual recipes should only have one product",
"if": {
"properties": {
"ProducedIn": {
"contains": {
"const": "manual"
}
}
}
},
"then": {
"properties": {
"Products": {
"maxItems": 1
}
}
}
}
],
"patternProperties": {
"schema": {
"$comment": "Allow the $schema line in files",
"type": "string"
},
"comment": {
"$comment": "Allow the $comment line in files",
"type": "string"
}
}
}

View file

@ -0,0 +1,23 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://raw.githubusercontent.com/budak7273/ContentLib_Documentation/main/JsonSchemas/CL_ItemAmount.json",
"title": "Item Amount Structure",
"description": "Respresents a quantity of an item",
"$comment": "This file does not represent ContentLib asset type! It's a resource used by other JSON Schemas.",
"type": "object",
"additionalProperties": false,
"required": [
"Item",
"Amount"
],
"properties": {
"Item": {
"$ref": "./CL_NameResolvedAsset.json"
},
"Amount": {
"type": "integer",
"minimum": 0,
"description": "Integer specifying the quantity of item(s). Remember that remember that 1000 fluid units displays as 1 Liter ingame."
}
}
}

View file

@ -0,0 +1,9 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://raw.githubusercontent.com/budak7273/ContentLib_Documentation/main/JsonSchemas/CL_NameResolvedAsset.json",
"title": "ContentLib Name Resolved Asset",
"description": "An asset that can be specified either with a full path, or a shortened name for ContentLib to resolve. See more info here: https://docs-dev.ficsit.app/contentlib/latest/BackgroundInfo/AutomaticNameResolving.html",
"$comment": "This file does not represent ContentLib asset type! It's a resource used by other JSON Schemas. Please read https://docs-dev.ficsit.app/contentlib/latest/BackgroundInfo/AutomaticNameResolving.html",
"type": "string",
"minLength": 1
}