Files
notarius/internal/core/config/operational_document.go

191 lines
7.2 KiB
Go

package config
import (
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
configkit "go.maximumdirect.net/configkit"
)
func resolveDocumentOperations(document *Document, resolution *configkit.Resolution, state *documentLoadState) error {
if document == nil {
return nil
}
normalizeDocumentState(document, resolution)
normalizeDocumentPaths(document, resolution, state)
normalizeDocumentPipelines(document, resolution, state)
if document.Concurrency != nil {
document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers,
configkit.Keys("concurrency", "stage_workers"), "stage worker identifier",
documentNormalizer{Resolution: resolution}, state)
}
deriveDocumentExtractWorkers(document, resolution, state)
return nil
}
func normalizeDocumentState(document *Document, resolution *configkit.Resolution) {
if document.Output != nil {
normalizeDocumentString(resolution, configkit.Keys("output", "directory"), &document.Output.Directory, "trimmed output directory")
}
if document.Cache != nil {
if document.Cache.ChunkPlans != nil {
normalizeDocumentString(resolution, configkit.Keys("cache", "chunk_plans", "directory"), &document.Cache.ChunkPlans.Directory, "trimmed chunk-plan cache directory")
normalizeDocumentString(resolution, configkit.Keys("cache", "chunk_plans", "mode"), &document.Cache.ChunkPlans.Mode, "trimmed chunk-plan cache mode")
}
if document.Cache.Checkpoints != nil {
normalizeDocumentString(resolution, configkit.Keys("cache", "checkpoints", "directory"), &document.Cache.Checkpoints.Directory, "trimmed checkpoint cache directory")
}
}
if document.Debug != nil {
normalizeDocumentString(resolution, configkit.Keys("debug", "directory"), &document.Debug.Directory, "trimmed debug directory")
}
}
func normalizeDocumentString(resolution *configkit.Resolution, path configkit.Path, target **string, reason string) {
if target == nil || *target == nil || !resolution.IsExplicit(path) {
return
}
configkit.Normalize(resolution, path, target, documentValue(strings.TrimSpace(**target)), reason)
}
func deriveDocumentExtractWorkers(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
if document.Concurrency == nil || document.Concurrency.TotalLLM == nil {
return
}
if document.Concurrency.StageWorkers == nil {
document.Concurrency.StageWorkers = make(map[string]int)
}
path := configkit.Keys("concurrency", "stage_workers", "extract")
// Presence belongs to the input spelling, not the key produced by trimming.
// A moved explicit count must never be replaced by a derived default.
if resolution.IsExplicit(state.originalPath(path)) {
return
}
workers := document.Concurrency.StageWorkers
value := workers["extract"]
configkit.Default(resolution, path, &value, *document.Concurrency.TotalLLM,
"derived extraction workers from total LLM concurrency", configkit.Keys("concurrency", "total_llm"))
workers["extract"] = value
}
func validateDocumentOperations(document *Document, validation *configkit.Validation) {
if document == nil {
return
}
validateDocumentConcurrency(document.Concurrency, validation)
validateDocumentState(document, validation)
validateDocumentPipelines(document, validation)
}
func validateDocumentConcurrency(concurrency *ConcurrencyDocument, validation *configkit.Validation) {
if concurrency == nil {
validation.Error(configkit.Keys("concurrency"), "must not be null")
return
}
totalPath := configkit.Keys("concurrency", "total_llm")
if concurrency.TotalLLM == nil {
validation.Error(totalPath, "must not be null")
return
}
totalValid := true
if err := validateTotalLLMConcurrency(*concurrency.TotalLLM); err != nil {
validation.Error(totalPath, err.Error())
totalValid = false
}
keys := make([]string, 0, len(concurrency.StageWorkers))
for key := range concurrency.StageWorkers {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if err := validateStageWorkerKey(key); err != nil {
validation.Error(configkit.Keys("concurrency", "stage_workers").Key(key), err.Error())
}
}
if !totalValid {
return
}
if workers, ok := concurrency.StageWorkers["extract"]; ok {
if err := validateExtractWorkerCount(workers, *concurrency.TotalLLM); err != nil {
validation.Error(configkit.Keys("concurrency", "stage_workers", "extract"), err.Error())
}
}
}
func validateDocumentState(document *Document, validation *configkit.Validation) {
if document.Output == nil || document.Output.Directory == nil {
validation.Error(configkit.Keys("output", "directory"), "must not be empty")
} else if err := validateRequiredPath("output.directory", *document.Output.Directory); err != nil {
validation.Error(configkit.Keys("output", "directory"), err.Error())
}
if document.Cache == nil || document.Cache.ChunkPlans == nil || document.Cache.ChunkPlans.Mode == nil {
validation.Error(configkit.Keys("cache", "chunk_plans", "mode"), "must be a supported cache mode")
} else if err := validateChunkPlanCacheMode(*document.Cache.ChunkPlans.Mode); err != nil {
validation.Error(configkit.Keys("cache", "chunk_plans", "mode"), err.Error())
}
if document.Cache != nil && document.Cache.ChunkPlans != nil && document.Cache.ChunkPlans.Directory != nil {
if err := validateOptionalPath("cache.chunk_plans.directory", *document.Cache.ChunkPlans.Directory); err != nil {
validation.Error(configkit.Keys("cache", "chunk_plans", "directory"), err.Error())
}
}
if document.Cache != nil && document.Cache.Checkpoints != nil && document.Cache.Checkpoints.Directory != nil {
if err := validateOptionalPath("cache.checkpoints.directory", *document.Cache.Checkpoints.Directory); err != nil {
validation.Error(configkit.Keys("cache", "checkpoints", "directory"), err.Error())
}
}
if document.Debug == nil || document.Debug.Directory == nil {
validation.Error(configkit.Keys("debug", "directory"), "must not be empty")
} else if err := validateRequiredPath("debug.directory", *document.Debug.Directory); err != nil {
validation.Error(configkit.Keys("debug", "directory"), err.Error())
}
}
func validateTotalLLMConcurrency(total int) error {
if total <= 0 {
return fmt.Errorf("total LLM concurrency must be greater than zero")
}
return nil
}
func validateStageWorkerKey(key string) error {
if strings.TrimSpace(key) == "" {
return fmt.Errorf("concurrency.stage_workers key must not be empty")
}
if key != "extract" {
return fmt.Errorf("concurrency.stage_workers key %q is not supported", key)
}
return nil
}
func validateExtractWorkerCount(workers, total int) error {
if workers < 1 || workers > total {
return fmt.Errorf("concurrency.stage_workers.extract must be between 1 and concurrency.total_llm (%d)", total)
}
return nil
}
func validateRequiredPath(name, value string) error {
if strings.TrimSpace(value) == "" {
return fmt.Errorf("%s must not be empty", name)
}
return validateOptionalPath(name, value)
}
func validateOptionalPath(name, value string) error {
if strings.ContainsRune(value, '\x00') {
return fmt.Errorf("%s must not contain NUL", name)
}
return nil
}
func validateChunkPlanCacheMode(value string) error {
if _, err := pipeline.ParseChunkCacheMode(value); err != nil {
return fmt.Errorf("cache.chunk_plans.mode: %w", err)
}
return nil
}