full processing stage done

Common/qdr.fnd.core/Thread/AbstractWorker.cs

@@ -4,6 +4,7 @@ using System.Threading.Tasks;
 using Quadarax.Foundation.Core.Logging;
 using Quadarax.Foundation.Core.Object;
 using Quadarax.Foundation.Core.Thread.Extensions;
+using Quadarax.Foundation.Core.Value.Extensions;
 
 namespace Quadarax.Foundation.Core.Thread
 {
@@ -15,6 +16,7 @@ namespace Quadarax.Foundation.Core.Thread
         private ILog _log;
         private TimeSpan _delay;
         private CancellationTokenSource _cancel;
+        private ILogger _logger;
         #endregion
 
         #region *** Properties ***
@@ -47,6 +49,7 @@ namespace Quadarax.Foundation.Core.Thread
             Context = context;
             _log = logger?.GetLogger(GetType());
             Timeout = TimeSpan.Zero;
+            _logger = logger;
         }
         #endregion
 
@@ -100,6 +103,14 @@ namespace Quadarax.Foundation.Core.Thread
         {
             throw new NotImplementedException("AbstractWorker.Do() must be overriden.");
         }
+        protected virtual void OnTimeout(IWorkerContext context)
+        {
+        }
+
+        protected virtual bool OnError(IWorkerContext wkContext, Exception exception)
+        {
+            return true;
+        }
 
         protected void Log(LogSeverityEnum severity, string message)
         {
@@ -118,6 +129,10 @@ namespace Quadarax.Foundation.Core.Thread
             _log = null;
         }
 
+        protected ILogger GetLogger()
+        {
+            return _logger;
+        }
         #endregion
 
         #region *** Private Operations ***
@@ -134,10 +149,17 @@ namespace Quadarax.Foundation.Core.Thread
                 else
                     await wk.Do(wk.Context).TimeoutAfter(Timeout);
             }
+            catch (TimeoutException e)
+            {
+                wk.Log(LogSeverityEnum.Error, $"Worker '{Name}' timeout ({Timeout.ToReadableString()}).", e);
+                OnTimeout(wk.Context);
+            }
             catch (Exception e)
             {
                 wk.Log(LogSeverityEnum.Error, $"Worker '{Name}' failed.", e);
-                throw;
+                var shouldThrow = OnError(wk.Context,e);
+                if (shouldThrow)
+                    throw;
             }
             finally
             {

Common/qdr.fnd.core/Thread/ContinousWorker.cs

@@ -14,13 +14,18 @@ namespace Quadarax.Foundation.Core.Thread
         #endregion
 
         #region *** Constructors ***
-        public ContinousWorker(Action body = null) : base()
+        public ContinousWorker(Action body = null, IWorkerContext context = null) : base(context)
         {
             _body = body;
         }
         public ContinousWorker(string workerName, ILogger logger = null) : base(workerName,null, logger)
         {
         }
+
+        public ContinousWorker(string workerName, IWorkerContext context, ILogger logger = null) : base(workerName,context , logger)
+        {
+        }
+
         #endregion
 
         #region *** Public Operations ***

ProcessServer/Business/Enums/ScenarioAcceptFilterCodeEnum.cs

@@ -0,0 +1,15 @@
+namespace BO.ProcessServer.Business.Enums
+{
+    public enum ScenarioAcceptFilterCodeEnum
+    {
+        PresetWatermarkIs,
+        PresetQualityEq,
+        PresetQualityLt,
+        PresetQualityGt,
+        PresetProcessProfileEq,
+        PresetProcessProfileStarts,
+        PresetProcessProfileEnds,
+        PresetProcessProfileContains,
+        PresetTestIs
+    }
+}

ProcessServer/Business/Pool.cs

@@ -26,6 +26,7 @@ namespace BO.ProcessServer.Business
 
         #region *** Constants ***
         private const string CS_ALL_FILES = "*.*";
+        private const string CS_ALL_FILES_WOEXT = "*";
         #endregion
 
         #region *** Private fields ***
@@ -80,13 +81,27 @@ namespace BO.ProcessServer.Business
             return fullFileName;
         }
 
-        public async Task<IEnumerable<string>> GetSinglePendingDocumentFilesAsync()
+        public async Task<Stream> PopDocumentAsync(string fullFileName)
+        {
+            var memory = new MemoryStream();
+            await using (var sw = _fileSystem.File.OpenRead(fullFileName))
+            {
+                await sw.CopyToAsync(memory);
+                await memory.FlushAsync();
+                _log.Log(LogSeverityEnum.Debug, $"Pool: File '{fullFileName}' pop.");
+
+            }
+            memory.Seek(0, SeekOrigin.Begin);
+            return memory;
+        }
+
+        public async Task<Tuple<long, IEnumerable<string>>> GetSinglePendingDocumentFilesAsync()
         {
             var first = GetPoolBranch(PoolTypeEnum.Pending).EnumerateFiles(CS_ALL_FILES, SearchOption.TopDirectoryOnly).FirstOrDefault();
-            if (first == null) return new List<string>();
+            if (first == null) return new Tuple<long, IEnumerable<string>>(-1, new List<string>());
 
             var documentId = GetDocumentIdFromFileName(first.Name);
-            return GetDocumentFiles(PoolTypeEnum.Pending, documentId);
+            return new Tuple<long, IEnumerable<string>>(documentId, GetDocumentFiles(PoolTypeEnum.Pending, documentId));
         }
 
         public long GetDocumentIdFromFileName(string fileName)
@@ -100,19 +115,38 @@ namespace BO.ProcessServer.Business
             return long.Parse(fileNameOnly.Substring(0, pos));
         }
 
-        public IEnumerable<string> GetDocumentsCount(PoolTypeEnum poolBranchFrom)
+        public int GetDocumentsCount(PoolTypeEnum poolBranchFrom)
+        {
+            return GetPoolBranch(poolBranchFrom).GetFiles(CS_ALL_FILES, SearchOption.TopDirectoryOnly).Select(x=>x.FullName).Count();
+        }
+        public long GetDocumentsSize(PoolTypeEnum poolBranchFrom)
         {
-            return GetPoolBranch(poolBranchFrom).GetFiles(CS_ALL_FILES, SearchOption.TopDirectoryOnly).Select(x=>x.FullName);
+            var files = GetPoolBranch(poolBranchFrom).GetFiles(CS_ALL_FILES, SearchOption.TopDirectoryOnly);
+            return files.Aggregate<IFileInfo, long>(0, (current, file) => current + file.Length);
+        }
+
+        public IEnumerable<long> GetDocumentIdsFromOldest(PoolTypeEnum poolBranchFrom)
+        {
+            var result = new List<long>();
+            var files = GetPoolBranch(poolBranchFrom).GetFiles(CS_ALL_FILES, SearchOption.TopDirectoryOnly).OrderByDescending(x=>x.LastWriteTime);
+            foreach (var file in files)
+            {
+                var docId = GetDocumentIdFromFileName(file.FullName);
+                if (!result.Contains(docId))
+                    result.Add(docId);
+            }
+
+            return result;
         }
 
-        public IEnumerable<string> GetDocumentFiles(PoolTypeEnum poolBranchFrom, long documentId)
+        public IEnumerable<string> GetDocumentFiles(PoolTypeEnum poolBranchFrom, long documentId, string alternativeExt = null)
         {
-            return GetPoolBranch(poolBranchFrom).GetFiles(TranslateSearchPattern(documentId), SearchOption.TopDirectoryOnly).Select(x=>x.FullName);
+            return GetPoolBranch(poolBranchFrom).GetFiles(TranslateSearchPattern(documentId,alternativeExt), SearchOption.TopDirectoryOnly).Select(x=>x.FullName);
         }
 
-        public IEnumerable<string> MoveDocumentFiles(PoolTypeEnum poolBranchFrom, PoolTypeEnum poolBranchTo, long documentId)
+        public IEnumerable<string> MoveDocumentFiles(PoolTypeEnum poolBranchFrom, PoolTypeEnum poolBranchTo, long documentId, string alternativeExt = null)
         {
-            var files = GetDocumentFiles(poolBranchFrom, documentId).ToArray();
+            var files = GetDocumentFiles(poolBranchFrom, documentId, alternativeExt).ToArray();
             var tasks = new List<Task>();
             foreach (var file in files)
             {
@@ -164,8 +198,11 @@ namespace BO.ProcessServer.Business
             return $"{documentId:D8}_{originalFileName}";
         }
 
-        private string TranslateSearchPattern(long documentId)
+        private string TranslateSearchPattern(long documentId,string alternativeExt = null)
         {
+            var search = CS_ALL_FILES;
+            if (!string.IsNullOrEmpty(alternativeExt))
+                search = CS_ALL_FILES_WOEXT + alternativeExt;
             return $"{documentId:D8}_{CS_ALL_FILES}";
         }
 

ProcessServer/Business/Scenarios/Scenarios.cs

@@ -0,0 +1,111 @@
+using System;
+using System.Diagnostics;
+using System.Linq;
+using BO.AppServer.Metadata.Dto;
+using BO.ProcessServer.Business.Enums;
+using BO.ProcessServer.Options;
+using Quadarax.Foundation.Core.Logging;
+
+
+namespace BO.ProcessServer.Business.Scenarios
+{
+    internal class Scenarios
+    {
+        private Configuration _configuration;
+        private ILog _log;
+
+        public Scenarios(Configuration configuration, ILogger logger)
+        {
+            if (logger == null) throw new ArgumentNullException(nameof(logger));
+            _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
+            _log = logger.GetLogger(GetType());
+        }
+        
+        public bool ExistsScenario(DocumentRDto docInfo)
+        {
+            if (docInfo == null) throw new ArgumentNullException(nameof(docInfo));
+            return _configuration.System.Scenarios.Any(x => MatchScenario(x, docInfo));
+        }
+
+        public Configuration.CfgScenario FindScenario(DocumentRDto docInfo)
+        {
+            return _configuration.System.Scenarios.FirstOrDefault(x => MatchScenario(x, docInfo));
+        }
+
+
+        private bool MatchScenario(Configuration.CfgScenario scenario, DocumentRDto document)
+        {
+
+            var match = scenario.AcceptFiltersIfAll;
+            foreach (var filter in scenario.AcceptFilters)
+            {
+
+                switch (filter.Code)
+                {
+                        case ScenarioAcceptFilterCodeEnum.PresetWatermarkIs:
+                            if (scenario.AcceptFiltersIfAll)
+                                match = match && bool.Parse(filter.Value) == document.PresetWatermark;
+                            else
+                                match = bool.Parse(filter.Value) == document.PresetWatermark;
+                            break;
+                        case ScenarioAcceptFilterCodeEnum.PresetQualityEq:
+                            if (scenario.AcceptFiltersIfAll)
+                                match = match && int.Parse(filter.Value) == document.PresetCreditCost;
+                            else
+                                match = int.Parse(filter.Value) == document.PresetCreditCost;
+                            break;
+                        case ScenarioAcceptFilterCodeEnum.PresetQualityLt:
+                            if (scenario.AcceptFiltersIfAll)
+                                match = match && int.Parse(filter.Value) > document.PresetCreditCost;
+                            else
+                                match = int.Parse(filter.Value) > document.PresetCreditCost;
+                            break;
+                        case ScenarioAcceptFilterCodeEnum.PresetQualityGt:
+                            if (scenario.AcceptFiltersIfAll)
+                                match = match && int.Parse(filter.Value) < document.PresetCreditCost;
+                            else
+                                match = int.Parse(filter.Value) < document.PresetCreditCost;
+                            break;
+                        case ScenarioAcceptFilterCodeEnum.PresetProcessProfileEq:
+                            if (scenario.AcceptFiltersIfAll)
+                                match = match && document.PresetProcessProfile == filter.Value;
+                            else
+                                match = document.PresetProcessProfile == filter.Value;
+                            break;
+                        case ScenarioAcceptFilterCodeEnum.PresetProcessProfileStarts:
+                            if (scenario.AcceptFiltersIfAll)
+                                match = match && document.PresetProcessProfile.StartsWith(filter.Value);
+                            else
+                                match = document.PresetProcessProfile.StartsWith(filter.Value);
+                            break;
+                        case ScenarioAcceptFilterCodeEnum.PresetProcessProfileEnds:
+                            if (scenario.AcceptFiltersIfAll)
+                                match = match && document.PresetProcessProfile.EndsWith(filter.Value);
+                            else
+                                match = document.PresetProcessProfile.EndsWith(filter.Value);
+                            break;
+                        case ScenarioAcceptFilterCodeEnum.PresetProcessProfileContains:
+                            if (scenario.AcceptFiltersIfAll)
+                                match = match && document.PresetProcessProfile.Contains(filter.Value);
+                            else
+                                match = document.PresetProcessProfile.Contains(filter.Value);
+                            break;
+                        case ScenarioAcceptFilterCodeEnum.PresetTestIs:
+                            if (scenario.AcceptFiltersIfAll)
+                                match = match && bool.Parse(filter.Value) == document.PresetTest;
+                            else
+                                match = bool.Parse(filter.Value) == document.PresetTest;
+                            break;
+                        default:
+                            throw new NotSupportedException($"Not supported code '{filter.Code}'");
+                }
+
+                if (!scenario.AcceptFiltersIfAll && match)
+                    break;
+                
+            }
+            return match;
+        }
+
+    }
+}

ProcessServer/Business/Server.cs

@@ -21,7 +21,7 @@ namespace BO.ProcessServer.Business
         #region *** Private fields ***
         private Configuration _configuration;
         private IFileSystem _fileSystem;
-        private IList<ServerProcess> _processes;
+        private IList<SingleDocumentCalculationWorker> _processes;
         protected ILog Log { get; }
         protected ILog LogLifecycle { get; }
         protected Statistics Statistics { get; }
@@ -46,18 +46,18 @@ namespace BO.ProcessServer.Business
             if (logger == null)
                 throw new ArgumentNullException(nameof(logger));
             Log = logger.GetLogger(typeof(Server));
-            LogLifecycle = logger.GetLogger("Lifecycle");
+            LogLifecycle = logger.GetLogger(Constants.CS_LOG_LOGGER_LIFECYCLE);
             Statistics = new Statistics();
 
             ValidateConfiguration();
             
-            _processes = new List<ServerProcess>();
+            _processes = new List<SingleDocumentCalculationWorker>();
 
             _connection = new Connector(_configuration, _fileSystem);
 
             _pool = new Pool(_configuration, _fileSystem, logger);
 
-            _wkPull = new PullDocumentsWorker(_connection, _configuration.System.PoolPullInterval, _fileSystem, Statistics, _pool, logger);
+            _wkPull = new PullDocumentsWorker(_connection, _configuration.System.PoolPullInterval, _fileSystem,_configuration, Statistics, _pool, logger);
             _wkRetention = new RetentionWorker(new CfgCtx(_configuration, Statistics, _pool));
             _wkProcessing = new ProcessingDocumentsWorker(_configuration, _processes, _connection, _fileSystem, Statistics, _pool, logger);
         }
@@ -183,6 +183,17 @@ namespace BO.ProcessServer.Business
             EnsureDirectory(_configuration.System.ScenarioPathBase);
             if (_fileSystem.Directory.GetFiles(TranslatePath(_configuration.System.ScenarioPathBase), "*.*").Length == 0)
                 LogWarn($"Scenario directory '{TranslatePath(_configuration.System.ScenarioPathBase)}' is empty.");
+            var missingCommands = new List<string>();
+            foreach (var scenario in _configuration.System.Scenarios)
+            {
+                var fileName = _fileSystem.Path.Combine(TranslatePath(_configuration.System.ScenarioPathBase), scenario.ShellCommand);
+                if (!_fileSystem.File.Exists(fileName))
+                    missingCommands.Add(scenario.ShellCommand);
+            }
+
+            if (missingCommands.Count > 0)
+                throw new FileNotFoundException(
+                    $"Following referenced command files missing in scenario directory '{TranslatePath(_configuration.System.ScenarioPathBase)}': {string.Join(",", missingCommands)}");
 
             LogInfo("Configuration validated.");
         }

ProcessServer/Business/ServerProcess.cs

@@ -1,16 +0,0 @@
-using System;
-using BO.ProcessServer.Business.Enums;
-using Quadarax.Foundation.Core.Thread;
-
-namespace BO.ProcessServer.Business
-{
-    internal class ServerProcess : ContinousWorker
-    {
-        public string ShellCommand { get; set; }
-        public ServerProcessStateEnum Status { get; set; }
-        public DateTime Started { get; set; }
-        public DateTime Finished { get; set; }
-
-
-    }
-}

ProcessServer/Business/Workers/Contexts/BaseCtx.cs

@@ -1,5 +1,6 @@
 using System;
 using BO.ProcessServer.Business.StatisticCounters;
+using BO.ProcessServer.Options;
 using Quadarax.Foundation.Core.Thread;
 
 namespace BO.ProcessServer.Business.Workers.Contexts
@@ -9,10 +10,13 @@ namespace BO.ProcessServer.Business.Workers.Contexts
         public Statistics Statistics { get; }
         public Pool Pool { get; }
 
-        protected BaseCtx(Statistics statistics, Pool pool)
+        public Configuration Configuration { get; }
+
+        protected BaseCtx(Configuration configuration, Statistics statistics, Pool pool)
         {
             Statistics = statistics ?? throw new ArgumentNullException(nameof(statistics));
             Pool = pool ?? throw new ArgumentNullException(nameof(pool));
+            Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
         }
     }
 }

ProcessServer/Business/Workers/Contexts/CalculationCtx.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using BO.ProcessServer.Business.StatisticCounters;
+using BO.ProcessServer.Options;
+using Connector.PS;
+
+namespace BO.ProcessServer.Business.Workers.Contexts
+{
+    internal class CalculationCtx : BaseCtx
+    {
+
+        public long DocumentId { get; private set; }
+        public IList<string> InputFiles { get; private set; }
+        public IList<string> OutputFiles { get; private set; }
+
+        public IConnectionPS Connection { get; private set; }
+        public Process Process { get; set; }
+
+        public CalculationCtx(IConnectionPS connection, long documentId, IList<string> files, Configuration configuration, Statistics statistics, Pool pool) : base(configuration, statistics, pool)
+        {
+            DocumentId = documentId;
+            InputFiles = files ?? throw new ArgumentNullException(nameof(files));
+            Connection = connection ?? throw new ArgumentNullException(nameof(connection));
+            OutputFiles = new List<string>();
+
+        }
+    }
+}

ProcessServer/Business/Workers/Contexts/CfgCtx.cs

@@ -1,16 +1,13 @@
-using System;
-using BO.ProcessServer.Business.StatisticCounters;
+using BO.ProcessServer.Business.StatisticCounters;
 using BO.ProcessServer.Options;
 
 namespace BO.ProcessServer.Business.Workers.Contexts
 {
     internal class CfgCtx : BaseCtx
     {
-        public Configuration Configuration { get; }
 
-        public CfgCtx(Configuration configuration, Statistics statistics, Pool pool) : base(statistics, pool)
+        public CfgCtx(Configuration configuration, Statistics statistics, Pool pool) : base(configuration, statistics, pool)
         {
-            Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
         }
     }
 }

ProcessServer/Business/Workers/Contexts/ProcessingDocumentsCtx.cs

@@ -10,12 +10,12 @@ namespace BO.ProcessServer.Business.Workers.Contexts
     internal class ProcessingDocumentsCtx : CfgCtx
     {
 
-        public IList<ServerProcess> Processes { get; }
+        public IList<SingleDocumentCalculationWorker> Processes { get; }
         public IConnectionPS Connection { get; set; }
         public IFileSystem FileSystem { get; set; }
 
 
-        public ProcessingDocumentsCtx(Configuration configuration,IList<ServerProcess> processes,IConnectionPS connection, IFileSystem fileSystem, Statistics statistics, Pool pool) : base(configuration, statistics, pool)
+        public ProcessingDocumentsCtx(Configuration configuration,IList<SingleDocumentCalculationWorker> processes,IConnectionPS connection, IFileSystem fileSystem, Statistics statistics, Pool pool) : base(configuration, statistics, pool)
         {
             Processes = processes ?? throw new ArgumentNullException(nameof(processes));
             Connection = connection ?? throw new ArgumentNullException(nameof(connection));

ProcessServer/Business/Workers/Contexts/PullCtx.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO.Abstractions;
 using BO.ProcessServer.Business.StatisticCounters;
+using BO.ProcessServer.Options;
 using Connector.PS;
 
 namespace BO.ProcessServer.Business.Workers.Contexts
@@ -12,7 +13,7 @@ namespace BO.ProcessServer.Business.Workers.Contexts
         public IFileSystem FileSystem { get; set; }
         public IList<DocumentContent> Documents { get; }
 
-        public PullCtx(IConnectionPS connection, IFileSystem fileSystem, Statistics statistics, Pool pool) : base(statistics, pool)
+        public PullCtx(IConnectionPS connection, IFileSystem fileSystem, Configuration configuration, Statistics statistics, Pool pool) : base(configuration, statistics, pool)
         {
             Connection = connection ?? throw new ArgumentNullException(nameof(connection));
             FileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));

ProcessServer/Business/Workers/ProcessingDocumentsWorker.cs

@@ -1,21 +1,29 @@
 using System.Collections.Generic;
 using System.IO.Abstractions;
+using System.Linq;
 using System.Threading.Tasks;
+using BO.AppServer.Metadata.Enums;
+using BO.ProcessServer.Business.Enums;
 using BO.ProcessServer.Business.StatisticCounters;
 using BO.ProcessServer.Business.Workers.Contexts;
 using BO.ProcessServer.Options;
 using Connector.PS;
+using NLog;
 using Quadarax.Foundation.Core.Logging;
 using Quadarax.Foundation.Core.Thread;
+using ILogger = Quadarax.Foundation.Core.Logging.ILogger;
 
 namespace BO.ProcessServer.Business.Workers
 {
     internal class ProcessingDocumentsWorker: LoopWorker
     {
         private const string CS_WK_NAME = "ProcessingDocument";
-        public ProcessingDocumentsWorker(Configuration configuration,IList<ServerProcess> processes, IConnectionPS connection, IFileSystem fileSystem,Statistics statistics, Pool pool, ILogger logger) 
+        private ILogger _logger;
+
+        public ProcessingDocumentsWorker(Configuration configuration,IList<SingleDocumentCalculationWorker> processes, IConnectionPS connection, IFileSystem fileSystem,Statistics statistics, Pool pool, ILogger logger) 
             : base(CS_WK_NAME, new ProcessingDocumentsCtx(configuration, processes, connection, fileSystem, statistics, pool), logger)
         {
+            _logger = logger;
         }
 
         public ProcessingDocumentsWorker(string workerName) : base(workerName)
@@ -33,8 +41,24 @@ namespace BO.ProcessServer.Business.Workers
             if (!ctx.Connection.IsOpen)
                 ctx.Connection.Open(ctx.Configuration.ApiUser, ctx.Configuration.ApiUserPwd, ctx.Configuration.ApiMagic);
 
-                
+            while (ctx.Processes.Count(x => x.Status == ServerProcessStateEnum.Pending) <
+                   ctx.Configuration.System.ScenarioParallelThreads)
+            {
+                var docFiles = await ctx.Pool.GetSinglePendingDocumentFilesAsync();
+                if (docFiles.Item1 > -1)
+                {
+                    Log(LogSeverityEnum.Info, $"Processing document ({docFiles.Item1}) with files {string.Join(";", docFiles.Item2.Select(x => "'" + ctx.FileSystem.Path.GetFileName(x)) + "'")} ...");
 
+                    var docinfo = ctx.Connection.GetDocumentInfoAsync(docFiles.Item1);
+                    if (docinfo != null)
+                    {
+                        var process = new SingleDocumentCalculationWorker(ctx.Connection, ctx.FileSystem, docFiles.Item1, docFiles.Item2.ToList(), ctx.Configuration, ctx.Statistics, ctx.Pool, _logger);
+                        ctx.Processes.Add(process);
+                        process.Start();
+                        await ctx.Connection.SetDocumentStateAsync(docFiles.Item1, DocumentStatusEnum.Processing);
+                    }
+                }
+            }
         }
     }
 }

ProcessServer/Business/Workers/PullDocumentsWorker.cs

@@ -6,6 +6,7 @@ using System.Threading.Tasks;
 using BO.AppServer.Metadata.Enums;
 using BO.ProcessServer.Business.StatisticCounters;
 using BO.ProcessServer.Business.Workers.Contexts;
+using BO.ProcessServer.Options;
 using Connector.PS;
 using Quadarax.Foundation.Core.Logging;
 using Quadarax.Foundation.Core.Thread;
@@ -17,9 +18,9 @@ namespace BO.ProcessServer.Business.Workers
         #region *** Constants ***
         private const string CS_WK_NAME = "PullDocument";
         #endregion
-
+        
         #region *** Constructors ***
-        public PullDocumentsWorker(IConnectionPS connection, TimeSpan loopEveryTimeSpan, IFileSystem fileSystem, Statistics statistics,Pool pool, ILogger logger) : base(CS_WK_NAME, new PullCtx(connection, fileSystem, statistics, pool), logger)
+        public PullDocumentsWorker(IConnectionPS connection, TimeSpan loopEveryTimeSpan, IFileSystem fileSystem,Configuration configuration, Statistics statistics,Pool pool, ILogger logger) : base(CS_WK_NAME, new PullCtx(connection, fileSystem,configuration, statistics, pool), logger)
         {
             IterationsDelayBetween = loopEveryTimeSpan;
         }
@@ -39,27 +40,40 @@ namespace BO.ProcessServer.Business.Workers
         {
             var ctx = (PullCtx)context;
 
+            var scenarioMan = new Scenarios.Scenarios(ctx.Configuration, GetLogger());
+
+            // get all pending document and set lock status (pending->working)
             var documents = await ctx.Connection.GetDocumentsAsync(DocumentsScopeEnums.Pending);
             foreach (var document in documents)
             {
-                var downloadedFiles = new List<string>();
-                foreach (var art in document.Artifacts.Where(x=>x.Type == ArtifactTypeEnum.Input))
+                if (scenarioMan.ExistsScenario(document))
                 {
-                    await using (var content = await ctx.Connection.GetDocumentContentAsync(document.Id, art.Id))
+                    // scenario for document exists
+                    var downloadedFiles = new List<string>();
+                    foreach (var art in document.Artifacts.Where(x => x.Type == ArtifactTypeEnum.Input))
                     {
-                        var fileName = ctx.FileSystem.Path.ChangeExtension(art.Name, art.Extension);
-                        fileName = await ctx.Pool.PutDocumentAsync(document.Id, fileName, content);
-                        downloadedFiles.Add(fileName);
-                        Log(LogSeverityEnum.Info, $"Document '{document.Name}' [{document.Id}] file '{fileName}' downloaded. (size {content.Length} bytes)");
+                        await using (var content = await ctx.Connection.GetDocumentContentAsync(document.Id, art.Id))
+                        {
+                            var fileName = ctx.FileSystem.Path.ChangeExtension(art.Name, art.Extension);
+                            fileName = await ctx.Pool.PutDocumentAsync(document.Id, fileName, content);
+                            downloadedFiles.Add(fileName);
+                            Log(LogSeverityEnum.Info,
+                                $"Document '{document.Name}' [{document.Id}] file '{fileName}' downloaded. (size {content.Length} bytes)");
+                        }
                     }
-                }
 
-                if (!downloadedFiles.All(x => ctx.FileSystem.File.Exists(x)))
+                    if (!downloadedFiles.All(x => ctx.FileSystem.File.Exists(x)))
+                    {
+                        // not all  downloaded -> rollback
+                        ctx.Pool.DeleteDocumentFiles(Pool.PoolTypeEnum.Pending, document.Id);
+                        await ctx.Connection.SetDocumentStateAsync(document.Id, DocumentStatusEnum.New);
+                        Log(LogSeverityEnum.Warn, $"Document '{document.Name}' [{document.Id}] roll-backed.");
+                    }
+                }
+                else
                 {
-                    // not all  downloaded -> rollback
-                    ctx.Pool.DeleteDocumentFiles(Pool.PoolTypeEnum.Pending, document.Id);
-                    await ctx.Connection.SetDocumentStateAsync(document.Id, DocumentStatusEnum.New);
-                    Log(LogSeverityEnum.Warn, $"Document '{document.Name}' [{document.Id}] roll-backed.");
+                    // scenario for document doesn't exists => set document for pending (working->pending)
+                    await ctx.Connection.SetDocumentStateAsync(document.Id, DocumentStatusEnum.Pending);
                 }
             }
         }

ProcessServer/Business/Workers/PushDocumentsWorker.cs

@@ -1,10 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace BO.ProcessServer.Business.Workers
-{
-    class PushDocumentsWorker
-    {
-    }
-}

ProcessServer/Business/Workers/RetentionWorker.cs

@@ -1,13 +1,26 @@
-using Quadarax.Foundation.Core.Logging;
+using System.Linq;
+using System.Threading.Tasks;
+using BO.ProcessServer.Business.Workers.Contexts;
+using Quadarax.Foundation.Core.Logging;
 using Quadarax.Foundation.Core.Thread;
 
 namespace BO.ProcessServer.Business.Workers
 {
     public class RetentionWorker : LoopWorker
     {
+        #region *** Constants ***
         private const string CS_WK_NAME = "Retention";
+        #endregion
+        #region *** Private fields ***
+        #endregion
+
+        #region *** Properties ***
+        #endregion
+
+        #region *** Constructors ***
         public RetentionWorker(IWorkerContext context) : base(CS_WK_NAME, context)
         {
+            IterationsDelayBetween = ((CfgCtx)context).Configuration.System.Retention.Interval;
         }
 
         public RetentionWorker(string workerName, IWorkerContext context) : base(workerName, context)
@@ -21,5 +34,76 @@ namespace BO.ProcessServer.Business.Workers
         public RetentionWorker(string workerName, IWorkerContext context, ILogger logger = null) : base(workerName, context, logger)
         {
         }
+        #endregion
+
+        #region *** Iteration ***
+
+        protected override async Task DoIteration(IWorkerContext context)
+        {
+            var ctx = (CfgCtx)context;
+            if (!ctx.Configuration.System.Retention.Enabled)
+                return;
+            Log(LogSeverityEnum.Debug,"Retention iteration started...");
+            var maxFiles = ctx.Configuration.System.Retention.Items;
+            var maxSize = ctx.Configuration.System.Retention.Size;
+
+            var delDoneFiles = 0;
+            var delFailFiles = 0;
+            var delDoneSize = 0;
+            var delFailSize = 0;
+            if (maxFiles > 0)
+            {
+                // retention by files count
+                delDoneFiles = DeleteRetentionByCount(ctx,Pool.PoolTypeEnum.Done, maxFiles);
+                delFailFiles = DeleteRetentionByCount(ctx,Pool.PoolTypeEnum.Fail, maxFiles);
+
+            }
+
+            if (maxSize > 0)
+            {
+                // retention by size
+                delDoneSize = DeleteRetentionBySize(ctx, Pool.PoolTypeEnum.Done, maxSize);
+                delFailSize = DeleteRetentionBySize(ctx, Pool.PoolTypeEnum.Fail, maxSize);
+            }
+            Log(LogSeverityEnum.Debug,"Retention iteration end...");
+            Log(LogSeverityEnum.Info,$"Retention deleted {delDoneFiles+delDoneSize + delFailFiles + delFailSize} files -> Pool.Done: Ret.Files={delDoneFiles}, Ret.Size={delDoneSize}; Pool.Fail: Ret.Files={delFailFiles}, Ret.Size={delFailSize}");
+
+        }
+
+        #endregion
+
+
+        #region *** Overrides ***
+        #endregion
+
+        #region *** Private Operations ***
+
+        private int DeleteRetentionByCount(CfgCtx ctx,Pool.PoolTypeEnum repo, int maxFiles)
+        {
+            var total = 0;
+            while (ctx.Pool.GetDocumentsCount(repo) > maxFiles)
+            {
+                var docIds = ctx.Pool.GetDocumentIdsFromOldest(repo).ToArray();
+                if (docIds.Length > 0)
+                    total += ctx.Pool.DeleteDocumentFiles(repo, docIds.First()).Count();
+            }
+            return total;
+        }
+        private int DeleteRetentionBySize(CfgCtx ctx,Pool.PoolTypeEnum repo, long maxSize)
+        {
+            var total = 0;
+            while (ctx.Pool.GetDocumentsSize(repo) > maxSize)
+            {
+                var docIds = ctx.Pool.GetDocumentIdsFromOldest(repo).ToArray();
+                if (docIds.Length > 0)
+                    total += ctx.Pool.DeleteDocumentFiles(repo, docIds.First()).Count();
+            }
+
+            return total;
+        }
+        
+        #endregion
+
+        
     }
 }

ProcessServer/Business/Workers/SingleDocumentCalculationWorker.cs

@@ -0,0 +1,160 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO.Abstractions;
+using System.Threading.Tasks;
+using BO.AppServer.Metadata.Enums;
+using BO.ProcessServer.Business.Enums;
+using BO.ProcessServer.Business.StatisticCounters;
+using BO.ProcessServer.Business.Workers.Contexts;
+using BO.ProcessServer.Options;
+using Connector.PS;
+using Quadarax.Foundation.Core.IO.MimeTypes;
+using Quadarax.Foundation.Core.Logging;
+using Quadarax.Foundation.Core.Thread;
+
+namespace BO.ProcessServer.Business.Workers
+{
+    internal class SingleDocumentCalculationWorker : ContinousWorker
+    {
+        #region *** Constants ***
+        private const string CS_WK_NAME = "Document";
+        #endregion
+        #region *** Private fields ***
+        private IFileSystem _fileSystem;
+        #endregion
+
+        #region *** Properties ***
+        public string ShellCommand { get; set; }
+        public ServerProcessStateEnum Status { get; set; }
+        public DateTime Started { get; set; }
+        public DateTime Finished { get; set; }
+        #endregion
+
+        #region *** Constructors ***
+        public SingleDocumentCalculationWorker(IConnectionPS connection, IFileSystem fileSystem, long documentId, IList<string> files, Configuration configuration, Statistics statistics, Pool pool,
+            ILogger logger) : base($"{CS_WK_NAME}#{documentId:D8}", new CalculationCtx(connection, documentId, files, configuration, statistics, pool),
+            logger)
+        {
+            Status = ServerProcessStateEnum.Pending;
+            _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
+        }
+        #endregion
+
+        #region *** Iteration ***
+        protected override async Task Do(IWorkerContext context)
+        {
+            var ctx = (CalculationCtx)context;
+
+            Status = ServerProcessStateEnum.Running;
+            Started = DateTime.Now;
+
+            // get document metadata
+            var document = await ctx.Connection.GetDocumentInfoAsync(ctx.DocumentId);
+            
+            // find document proper scenario
+            var scenarioMan = new Scenarios.Scenarios(ctx.Configuration, GetLogger());
+            var scenario = scenarioMan.FindScenario(document);
+            if (scenario == null)
+            {
+                // scenario doesn't exists for document -> change status to pending and move files to Failed branch.
+                await ctx.Connection.SetDocumentStateAsync(ctx.DocumentId, DocumentStatusEnum.Pending);
+                ctx.Pool.MoveDocumentFiles(Pool.PoolTypeEnum.Working, Pool.PoolTypeEnum.Fail, ctx.DocumentId);
+                Status = ServerProcessStateEnum.FailInternal;
+            }
+            else
+            {
+                // scenario exists
+                var processInfo = new ProcessStartInfo();
+                processInfo.FileName = scenario.ShellCommand;
+                processInfo.WindowStyle = scenario.RunWindowsStyle;
+                processInfo.WorkingDirectory = ctx.Configuration.System.ScenarioPathBase;
+                processInfo.Arguments = TranslateArguments(scenario.ShellCommandArguments, ctx.InputFiles, document.PresetProcessProfile,document.PresetQuality, document.PresetWatermark,document.PresetTest);
+                ShellCommand = processInfo.FileName + " " + processInfo.Arguments;
+                ctx.Process = Process.Start(processInfo);
+                Log(LogSeverityEnum.Info, $"Starting process '{Name}.{ctx.Process?.ProcessName}' [Id:{ctx.Process?.Id}]: {ShellCommand}");
+                if (ctx.Process != null)
+                {
+                    ctx.Process.WaitForExit();
+                    Log(LogSeverityEnum.Info, $"End process '{Name}.{ctx.Process?.ProcessName}' [Id:{ctx.Process?.Id}]: Exit code {ctx.Process.ExitCode}");
+
+                    if (ctx.Process.ExitCode == scenario.ExitCodeSucc)
+                    {
+                        Status = ServerProcessStateEnum.Done;
+
+                    }else if (ctx.Process.ExitCode == scenario.ExitCodeFail)
+                    {
+                        Status = ServerProcessStateEnum.FailExit;
+                    }
+                    else
+                    {
+                        Log(LogSeverityEnum.Warn, $"Process '{Name}.{ctx.Process?.ProcessName}' has unhandled exitcode {ctx.Process.ExitCode}! Goes to fail.");
+                        Status = ServerProcessStateEnum.FailInternal;
+                    }
+                }
+                else
+                {
+                    Status = ServerProcessStateEnum.FailInternal;
+                }
+            }
+            Finished = DateTime.Now;
+
+            if (Status == ServerProcessStateEnum.Done)
+            {
+                var files = ctx.Pool.GetDocumentFiles(Pool.PoolTypeEnum.Working, document.Id, Constants.CS_POOL_OUTFILE_EXT);
+                foreach (var file in files)
+                {
+                    await using (var fs = await ctx.Pool.PopDocumentAsync(file))
+                    {
+                        await ctx.Connection.SetDocumentContentAsync(document.Id,
+                            _fileSystem.Path.GetFileNameWithoutExtension(file),
+                            MimeTypeUtil.GetMimeType(_fileSystem.Path.GetExtension(file)), false, fs);
+                        Log(LogSeverityEnum.Debug, $"Process '{Name}.{ctx.Process?.ProcessName}' file '{file}' uploaded.");
+                    }
+                }
+                await ctx.Connection.SetDocumentStateAsync(document.Id, DocumentStatusEnum.DoneOk);
+                ctx.Pool.MoveDocumentFiles(Pool.PoolTypeEnum.Working, Pool.PoolTypeEnum.Done, document.Id);
+            }
+            else if (Status == ServerProcessStateEnum.FailInternal || Status == ServerProcessStateEnum.FailExit ||
+                     Status == ServerProcessStateEnum.FailTimeout)
+            {
+                ctx.Pool.MoveDocumentFiles(Pool.PoolTypeEnum.Working, Pool.PoolTypeEnum.Fail, document.Id);
+            }
+            Log(LogSeverityEnum.Info, $"Process '{Name}.{ctx.Process?.ProcessName}' resolution state is '{Status}'.");
+        }
+
+        #endregion
+
+
+        #region *** Overrides ***
+        protected override void OnBeforeStart()
+        {
+            Status = ServerProcessStateEnum.Running;
+            Started = DateTime.Now;
+            base.OnBeforeStart();
+        }
+
+        protected override void OnTimeout(IWorkerContext context)
+        {
+            var ctx = (CalculationCtx)context;
+            ctx.Process.Dispose();
+            ctx.Process = null;
+            Finished = DateTime.Now;
+            Status = ServerProcessStateEnum.FailTimeout;
+        }
+        #endregion
+
+        #region *** Private Operations ***
+        private string TranslateArguments(string arguments,IEnumerable<string> files, string profile, int quality, bool isWatermark,bool isTest)
+        {
+            var result = arguments.Replace(Constants.CS_TAG_INPUT_FILES, string.Join(Constants.CS_TAG_INPUT_FILES_SEP, files),StringComparison.OrdinalIgnoreCase);
+            result = result.Replace(Constants.CS_TAG_P_PROFILE, profile, StringComparison.OrdinalIgnoreCase);
+            result = result.Replace(Constants.CS_TAG_P_TEST, isTest.ToString(), StringComparison.OrdinalIgnoreCase);
+            result = result.Replace(Constants.CS_TAG_P_WATERMARK, isWatermark.ToString(), StringComparison.OrdinalIgnoreCase);
+            result = result.Replace(Constants.CS_TAG_P_QUALITY, quality.ToString(), StringComparison.OrdinalIgnoreCase);
+            result = result.Replace(Constants.CS_TAG_P_OUT_EXT, Constants.CS_POOL_OUTFILE_EXT, StringComparison.OrdinalIgnoreCase);
+            return result;
+        }
+        #endregion
+    }
+}

ProcessServer/Constants.cs

@@ -5,5 +5,21 @@
     {
         public const string CS_CONFIGURATION_FILE = "processserver.json";
         public const string CS_NLOG_CONFIGURATION_FILE = "processserver-nlog.config";
+        public const string CS_LOG_LOGGER_LIFECYCLE = "Lifecycle";
+
+
+        public const string CS_TAG_BEGIN = "{";
+        public const string CS_TAG_END = "}";
+        public const string CS_TAG_CURRENT_DIR = CS_TAG_BEGIN + "CD" + CS_TAG_END;
+        public const string CS_TAG_INPUT_FILES = CS_TAG_BEGIN + "FILENAMES" + CS_TAG_END;
+        public const string CS_TAG_INPUT_FILES_SEP = ",";
+        public const string CS_TAG_P_PROFILE = CS_TAG_BEGIN + "PROFILE" + CS_TAG_END;
+        public const string CS_TAG_P_QUALITY = CS_TAG_BEGIN + "QUALITY" + CS_TAG_END;
+        public const string CS_TAG_P_TEST = CS_TAG_BEGIN + "ISTEST" + CS_TAG_END;
+        public const string CS_TAG_P_WATERMARK = CS_TAG_BEGIN + "ISWATERMARK" + CS_TAG_END;
+        public const string CS_TAG_P_OUT_EXT = CS_TAG_BEGIN + "OUT-EXT" + CS_TAG_END;
+
+        public const string CS_POOL_OUTFILE_EXT = ".out";
+
     }
 }

ProcessServer/Options/Configuration.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Text.Json.Serialization;
 using BO.ProcessServer.Business.Enums;
 
@@ -38,7 +39,7 @@ namespace BO.ProcessServer.Options
 
         public class CfgScenarioAcceptFilter
         {
-            [JsonPropertyName("code")] public string Code { get; set; }
+            [JsonPropertyName("code")] public ScenarioAcceptFilterCodeEnum Code { get; set; }
             [JsonPropertyName("value")] public string Value { get; set; }
         }
 
@@ -46,7 +47,11 @@ namespace BO.ProcessServer.Options
         {
             [JsonPropertyName("name")] public string Name { get; set; }
             [JsonPropertyName("cmd")] public string ShellCommand { get; set; }
+            [JsonPropertyName("run-windows-style")] public string  RunWindowsStyleRaw { get; set; }
+            public ProcessWindowStyle RunWindowsStyle => Enum.Parse<ProcessWindowStyle>(RunWindowsStyleRaw);
+            [JsonPropertyName("run-hidden")] public bool RunHidden { get; set; }
             [JsonPropertyName("args")] public string ShellCommandArguments { get; set; }
+            [JsonPropertyName("accept-filters-if-all")] public bool AcceptFiltersIfAll { get; set; }
             [JsonPropertyName("accept-filters")] public IEnumerable<CfgScenarioAcceptFilter> AcceptFilters { get; set; }
             [JsonPropertyName("timeout-interval")] public string TimeoutIntervalRaw { get; set; }
             public TimeSpan TimeoutInterval => TimeSpan.Parse(TimeoutIntervalRaw);

ProcessServer/processServer.json

@@ -17,7 +17,7 @@
 		"pool-path-fail": ".fail",
 		"pool-retention": {
 			"enabled": true,
-			"retention-": "24:00:00",
+			"retention-interval": "24:00:00",
 			"size": 1000000000,
 			"items": 100
 		},
@@ -32,14 +32,16 @@
 		"Scenarios": [
 			{
 				"name": "test",
+				"accept-filters-if-all": true,
 				"accept-filters": [
 					{
-						"code": "profile",
+						"code": "PresetProcessProfileEq",
 						"value": "test"
 					}
 				],
 				"cmd": "scenario-dummy.cmd",
-				"args": "{filenames} {billingplan}",
+				"run-windows-style": "Normal",
+        "args": "{filenames} {billingplan}",
 				"timeout-interval": "01:00:00",
 				"exitcode-succ": 0,
 				"exitcode-fail": 1