sanitizer

magefile.go

@@ -4,8 +4,12 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"unicode"
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
@@ -13,16 +17,17 @@ import (
 
 const (
 	zipName    = "DigitalStorageTweaks.zip"
-	filesToCopy = "./ContentLib ./DigitalStorageTweaks.uplugin"
+	contentDir = "./ContentLib"
+	pluginFile = "./DigitalStorageTweaks.uplugin"
 	targetDirs = "Windows WindowsServer LinuxServer"
 )
 
-// Default target to run when none is specified
 var Default = Package
 
 // Package creates the distribution zip file
 func Package() error {
-	mg.Deps(Clean)
+	mg.Deps(SanitizeContent)
+	defer Clean(false) // Clean temp dirs but keep zip
 
 	fmt.Println("Creating target directories...")
 	for _, dir := range []string{"Windows", "WindowsServer", "LinuxServer"} {
@@ -31,7 +36,7 @@ func Package() error {
 		}
 
 		fmt.Printf("Copying files to %s...\n", dir)
-		if err := sh.Run("cp", "-r", "./ContentLib", "./DigitalStorageTweaks.uplugin", dir+"/"); err != nil {
+		if err := sh.Run("cp", "-r", contentDir, pluginFile, dir+"/"); err != nil {
 			return fmt.Errorf("failed to copy files to %s: %v", dir, err)
 		}
 	}
@@ -40,15 +45,84 @@ func Package() error {
 	return sh.Run("7z", "a", "-r", zipName, "Windows/", "LinuxServer/", "WindowsServer/")
 }
 
-// Clean removes generated directories and zip file
+// Clean removes generated artifacts
-func Clean() error {
+func Clean(includeZip bool) error {
-	fmt.Println("Cleaning up...")
+	fmt.Println("Cleaning artifacts...")
-	toRemove := []string{"Windows", "WindowsServer", "LinuxServer", zipName}
+	paths := []string{"Windows", "WindowsServer", "LinuxServer"}
-
+	if includeZip {
-	for _, path := range toRemove {
+		paths = append(paths, zipName)
-		if err := os.RemoveAll(path); err != nil {
+	}
-			return fmt.Errorf("failed to remove %s: %v", path, err)
+	for _, path := range paths {
+		if err := sh.Rm(path); err != nil {
+			return err
 		}
 	}
 	return nil
 }
+
+// SanitizeContent cleans all text files of non-ASCII and ensures CRLF line endings
+func SanitizeContent() error {
+	fmt.Println("Sanitizing content files...")
+
+	// Process plugin file
+	if err := sanitizeFile(pluginFile); err != nil {
+		return fmt.Errorf("plugin file sanitization failed: %v", err)
+	}
+
+	// Process content directory
+	return filepath.Walk(contentDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil || info.IsDir() {
+			return err
+		}
+
+		// Skip binary files
+		switch filepath.Ext(path) {
+		case ".png", ".jpg", ".jpeg", ".bmp", ".gif", ".dds", ".tga", ".psd", ".fbx", ".uasset", ".umap":
+			return nil
+		default:
+			return sanitizeFile(path)
+		}
+	})
+}
+
+func sanitizeFile(path string) error {
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("failed to read %s: %v", path, err)
+	}
+
+	// First fix any encoding issues by converting to valid UTF-8
+	content = bytes.ToValidUTF8(content, []byte{})
+
+	// Remove non-ASCII characters but preserve standard whitespace
+	cleanContent := bytes.Map(func(r rune) rune {
+		switch {
+		case r > unicode.MaxASCII:
+			return -1 // Remove non-ASCII
+		case r == '\t', r == '\n', r == '\r':
+			return r // Keep standard whitespace
+		case r < 32 || r == 127:
+			return -1 // Remove control characters
+		default:
+			return r // Keep regular ASCII
+		}
+	}, content)
+
+	// Normalize line endings to CRLF
+	cleanContent = bytes.ReplaceAll(cleanContent, []byte("\r\n"), []byte("\n"))
+	cleanContent = bytes.ReplaceAll(cleanContent, []byte("\n"), []byte("\r\n"))
+
+	// Only write back if changes were made
+	if !bytes.Equal(content, cleanContent) {
+		fmt.Printf("Sanitizing %s\n", path)
+		fileInfo, err := os.Stat(path)
+		if err != nil {
+			return fmt.Errorf("failed to get file info for %s: %v", path, err)
+		}
+		if err := ioutil.WriteFile(path, cleanContent, fileInfo.Mode()); err != nil {
+			return fmt.Errorf("failed to write %s: %v", path, err)
+		}
+	}
+
+	return nil
+}