using Sonex.Library.WorkersCore; using Sonex.Data.Records; using System.ServiceProcess; using WorkerCore = Sonex.Library.WorkersCore.Worker; namespace Sonex.Worker.WebSync; internal sealed class WebSyncWindowsService : ServiceBase { private static readonly TimeSpan StatusReportInterval = TimeSpan.FromSeconds(30); private const string WorkerName = "Sonex.Worker.WebSync"; private const string WorkerTitle = "Website Product Data Import"; private const string WorkerDescription = "Imports product data from xenos.nl and synchronizes associated product images."; private const string WindowsServiceStartMode = "auto"; private const string OperationWorkerLifecycle = "WorkerLifecycle"; private const string OperationCycleExecution = "CycleExecution"; private const string OperationStaleLinkRecheck = "StaleLinkRecheck"; private const string OperationStaleRecordCleanup = "StaleRecordCleanup"; private const string OperationDictionaryValuesUpdate = "DictionaryValuesUpdate"; private const string OperationTrendingCollectionScan = "TrendingCollectionScan"; private const string OperationNieuwCollectionScan = "NieuwCollectionScan"; private const string OperationActiesCollectionScan = "ActiesCollectionScan"; private const string TargetImagesPathKey = "TARGET_IMAGES_PATH"; private const string SitemapUrlKey = "SITEMAP_URL"; private const string TrendingUrlKey = "TRENDING_URL"; private const string NieuwUrlKey = "NIEUW_URL"; private const string ActiesUrlKey = "ACTIES_URL"; private const string MaxParallelTasksKey = "MAX_PARALLEL_TASKS"; private const string RetryCountKey = "RETRY_COUNT"; private const string DownloadTimeoutSecondsKey = "DOWNLOAD_TIMEOUT_SECONDS"; private const string RetryDelaySecondsKey = "RETRY_DELAY_SECONDS"; private const string UpdateImagesKey = "UPDATE_IMAGES"; private readonly object _lifecycleLock = new(); private readonly ManualResetEventSlim _resumeSignal = new(initialState: true); private readonly WebSyncSitemapLoader _sitemapLoader = new(); private readonly WebSyncProductTaskRunner _taskRunner = new(); private readonly WebSyncProductLinksProcessor _linksProcessor; private readonly WebSyncCollectionLoader _collectionLoader = new(); private CancellationTokenSource? _stoppingCts; private CancellationTokenSource? _statusReportingCts; private Task? _runTask; private Task? _statusReportingTask; private bool _isConsoleMode; private DateTime _runStartedAtUtc; private WebSyncRunReport _runReport = new(); public WebSyncWindowsService() { ServiceName = WorkerName; CanStop = true; CanPauseAndContinue = true; AutoLog = false; _linksProcessor = new WebSyncProductLinksProcessor(_taskRunner); } public void RunConsole() { _isConsoleMode = true; OnStart([]); Task? runTask; lock (_lifecycleLock) { runTask = _runTask; } try { runTask?.Wait(); } catch (AggregateException) { } OnStop(); } protected override void OnStart(string[] args) { lock (_lifecycleLock) { if (_runTask is not null) return; WorkerCore.Name = WorkerName; WorkerCore.Title = WorkerTitle; WorkerCore.Description = WorkerDescription; WorkerCore.WorkType = WorkerWorkType.Interval; ConfigureWorkerParameters(); WorkerCore.Registration( ensureWindowsServiceInConsole: _isConsoleMode, serviceStartMode: WindowsServiceStartMode, serviceDescription: WorkerDescription); WorkerCore.WorkerStarted(); WorkerCore.UpdateActivity("Idle"); WorkerCore.UpdateStatus(); WorkerCore.LogInfo("Worker started.", OperationWorkerLifecycle); _runStartedAtUtc = DateTime.UtcNow; _runReport = new WebSyncRunReport(); _stoppingCts = new CancellationTokenSource(); _statusReportingCts = CancellationTokenSource.CreateLinkedTokenSource(_stoppingCts.Token); _resumeSignal.Set(); _runTask = Task.Run(() => RunAsync(_stoppingCts.Token)); _statusReportingTask = Task.Run(() => StatusReportingLoopAsync(_statusReportingCts.Token)); } } protected override void OnPause() { lock (_lifecycleLock) { _resumeSignal.Reset(); WorkerCore.WorkerPaused(); WorkerCore.UpdateStatus(); WorkerCore.LogInfo( $"Worker paused. Progress snapshot: {BuildProgressSnapshot()}.", OperationWorkerLifecycle); } } protected override void OnContinue() { lock (_lifecycleLock) { _resumeSignal.Set(); WorkerCore.WorkerResumed(); WorkerCore.UpdateStatus(); WorkerCore.LogInfo( $"Worker resumed. Progress snapshot: {BuildProgressSnapshot()}.", OperationWorkerLifecycle); } } protected override void OnStop() { Task? runTask; Task? statusReportingTask; CancellationTokenSource? cts; CancellationTokenSource? statusReportingCts; string stopProgressSnapshot = BuildProgressSnapshot(); lock (_lifecycleLock) { runTask = _runTask; statusReportingTask = _statusReportingTask; cts = _stoppingCts; statusReportingCts = _statusReportingCts; _runTask = null; _statusReportingTask = null; _stoppingCts = null; _statusReportingCts = null; _resumeSignal.Set(); } WorkerCore.LogInfo( $"Worker stop requested. Progress snapshot: {stopProgressSnapshot}.", OperationWorkerLifecycle); if (cts is not null) { try { cts.Cancel(); } finally { cts.Dispose(); } } if (statusReportingCts is not null) { try { statusReportingCts.Cancel(); } finally { statusReportingCts.Dispose(); } } try { runTask?.Wait(TimeSpan.FromSeconds(15)); } catch (AggregateException) { } try { statusReportingTask?.Wait(TimeSpan.FromSeconds(5)); } catch (AggregateException) { } WorkerCore.WorkerFinished(); WorkerCore.UpdateStatus(); TimeSpan runDuration = BuildRunDuration(); int processedTasks = _runReport.GetProcessedTasksForSpeed(); string finishSummary = _runReport.BuildSummaryReport(runDuration, processedTasks); string fullReport = _runReport.BuildFullReport(runDuration, processedTasks); Console.WriteLine(fullReport); WorkerCore.LogInfo(finishSummary, OperationWorkerLifecycle); } protected override void Dispose(bool disposing) { if (disposing) _resumeSignal.Dispose(); base.Dispose(disposing); } private async Task RunAsync(CancellationToken cancellationToken) { try { WorkerCore.LogInfo("Cycle started.", OperationCycleExecution); _resumeSignal.Wait(cancellationToken); await ExecuteCycleAsync(cancellationToken).ConfigureAwait(false); if (!cancellationToken.IsCancellationRequested) { WorkerCore.UpdateActivity("Completed"); WorkerCore.UpdateStatus(); WorkerCore.LogInfo("Cycle completed.", OperationCycleExecution); if (!_isConsoleMode) { _ = Task.Run(Stop); } } } catch (OperationCanceledException) { } catch (Exception ex) { WorkerCore.WorkerError("Cycle error"); WorkerCore.UpdateStatus(); Console.Error.WriteLine(ex.ToString()); string operation = ex is WebSyncOperationException operationException ? operationException.Operation : OperationCycleExecution; _runReport.RegisterException(operation, ex, "Cycle execution failed."); WorkerCore.LogError("Cycle execution failed.", ex, operation); } } private async Task StatusReportingLoopAsync(CancellationToken cancellationToken) { try { using var timer = new PeriodicTimer(StatusReportInterval); while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) { WorkerCore.UpdateStatus(); } } catch (OperationCanceledException) { } } private async Task ExecuteCycleAsync(CancellationToken cancellationToken) { DateTime cycleStartedAt = DateTime.Now; var settings = LoadRuntimeSettings(); await WaitIfPausedAsync(cancellationToken).ConfigureAwait(false); WorkerCore.UpdateActivity("Downloading sitemap"); WorkerCore.UpdateProgress(0, 0); WorkerCore.UpdateStatus(); Console.WriteLine($"Downloading sitemap from {settings.SitemapUrl}..."); var productLinks = await DownloadProductLinksAsync(settings, cancellationToken).ConfigureAwait(false); await WaitIfPausedAsync(cancellationToken).ConfigureAwait(false); CollectionImportData collectionImportData = await DownloadCollectionDataAsync(settings, cancellationToken).ConfigureAwait(false); IReadOnlyList allProductLinks = MergeProductLinks(productLinks, collectionImportData.ProductUrls); _runReport.SetSitemapLinksCount(allProductLinks.Count); await WaitIfPausedAsync(cancellationToken).ConfigureAwait(false); WorkerCore.UpdateProgress(0, allProductLinks.Count); WorkerCore.UpdateActivity("Processing links"); WorkerCore.UpdateStatus(); if (allProductLinks.Count > 0) { await ProcessProductLinksAsync( allProductLinks, settings, collectionImportData.Snapshot, cancellationToken).ConfigureAwait(false); } await RefreshStaleProductsAsync( cycleStartedAt, settings, collectionImportData.Snapshot, cancellationToken).ConfigureAwait(false); await WaitIfPausedAsync(cancellationToken).ConfigureAwait(false); await CleanupStaleProductsAsync(cycleStartedAt, cancellationToken).ConfigureAwait(false); await WaitIfPausedAsync(cancellationToken).ConfigureAwait(false); await UpdateDictionaryValuesAsync(cancellationToken).ConfigureAwait(false); WorkerCore.UpdateProgress(allProductLinks.Count, allProductLinks.Count); WorkerCore.UpdateActivity("Cycle completed"); WorkerCore.UpdateStatus(); } private async Task ProcessProductLinksAsync( IReadOnlyList productLinks, RuntimeSettings settings, WebSyncCollectionSnapshot collectionSnapshot, CancellationToken cancellationToken) { await _linksProcessor.ProcessAsync( productLinks, settings.TargetImagesPath, settings.UpdateImages, collectionSnapshot, settings.MaxParallelTasks, settings.RetryCount, settings.DownloadTimeoutSeconds, settings.RetryDelaySeconds, _runReport, WaitIfPausedAsync, cancellationToken).ConfigureAwait(false); } private async Task> DownloadProductLinksAsync( RuntimeSettings settings, CancellationToken cancellationToken) { return await _sitemapLoader.DownloadProductLinksAsync( settings.SitemapUrl, settings.RetryCount, settings.DownloadTimeoutSeconds, settings.RetryDelaySeconds, _runReport, WaitIfPausedAsync, cancellationToken).ConfigureAwait(false); } private async Task RefreshStaleProductsAsync( DateTime cycleStartedAt, RuntimeSettings settings, WebSyncCollectionSnapshot collectionSnapshot, CancellationToken cancellationToken) { WorkerCore.UpdateActivity("Loading stale links from database"); WorkerCore.UpdateStatus(); await WaitIfPausedAsync(cancellationToken).ConfigureAwait(false); var staleResult = await WorkerCore.ExecuteDatabaseListWithRetryAsync( ct => ProductWebInfoRecord.GetStaleCandidates(cycleStartedAt, ct), OperationStaleLinkRecheck, cancellationToken) .ConfigureAwait(false); if (!staleResult.Success) { string error = staleResult.ErrorMessage ?? "Unknown stale-links query error."; throw new WebSyncOperationException( OperationStaleLinkRecheck, $"Loading stale links failed. Threshold={cycleStartedAt:yyyy-MM-dd HH:mm:ss}. Error={error}", new InvalidOperationException(error)); } List staleLinks = BuildDistinctLinks(staleResult.Items); if (staleLinks.Count == 0) { WorkerCore.LogInfo( $"Stale links check skipped. No links found before {cycleStartedAt:yyyy-MM-dd HH:mm:ss}.", OperationStaleLinkRecheck); return; } WorkerCore.LogInfo( $"Stale links recheck started. Candidates={staleResult.Items.Count}, Links={staleLinks.Count}.", OperationStaleLinkRecheck); await WaitIfPausedAsync(cancellationToken).ConfigureAwait(false); WorkerCore.UpdateProgress(0, staleLinks.Count); WorkerCore.UpdateActivity("Rechecking stale links"); WorkerCore.UpdateStatus(); await ProcessProductLinksAsync( staleLinks, settings, collectionSnapshot, cancellationToken).ConfigureAwait(false); WorkerCore.LogInfo( $"Stale links recheck completed. Retried links={staleLinks.Count}.", OperationStaleLinkRecheck); } private static List BuildDistinctLinks( IReadOnlyList candidates) { var links = new List(candidates.Count); var seen = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var candidate in candidates) { string url = candidate.Url?.Trim() ?? string.Empty; if (string.IsNullOrWhiteSpace(url)) continue; if (!Uri.TryCreate(url, UriKind.Absolute, out _)) continue; if (seen.Add(url)) links.Add(url); } return links; } private static RuntimeSettings LoadRuntimeSettings() { return new RuntimeSettings { TargetImagesPath = GetParameterString(TargetImagesPathKey, @"c:\sonex\wwwroot\products\"), SitemapUrl = GetParameterString(SitemapUrlKey, "https://www.xenos.nl/storage/sitemap/default/sitemap.xml"), TrendingUrl = GetParameterString(TrendingUrlKey, "https://www.xenos.nl/trending"), NieuwUrl = GetParameterString(NieuwUrlKey, "https://www.xenos.nl/nieuw"), ActiesUrl = GetParameterString(ActiesUrlKey, "https://www.xenos.nl/acties"), MaxParallelTasks = GetParameterInt(MaxParallelTasksKey, 20, 1), RetryCount = GetParameterInt(RetryCountKey, 2, 0), DownloadTimeoutSeconds = GetParameterInt(DownloadTimeoutSecondsKey, 5, 1), RetryDelaySeconds = GetParameterInt(RetryDelaySecondsKey, 120, 0), UpdateImages = GetParameterBool(UpdateImagesKey, true) }; } private static string GetParameterString(string key, string defaultValue) { var parameter = WorkerCore.Parameters .FirstOrDefault(item => string.Equals(item.Parametr, key, StringComparison.OrdinalIgnoreCase)); return string.IsNullOrWhiteSpace(parameter?.Value) ? defaultValue : parameter.Value.Trim(); } private static int GetParameterInt(string key, int defaultValue, int minValue) { var value = GetParameterString(key, defaultValue.ToString()); if (!int.TryParse(value, out int parsed)) return defaultValue; return Math.Max(minValue, parsed); } private static bool GetParameterBool(string key, bool defaultValue) { var value = GetParameterString(key, defaultValue.ToString()); return bool.TryParse(value, out bool parsed) ? parsed : defaultValue; } private static string BuildProgressSnapshot() { string progressInfo = WorkerCore.ProgressInfo?.Trim() ?? string.Empty; if (!string.IsNullOrWhiteSpace(progressInfo)) return progressInfo; string progress = WorkerCore.Progress?.Trim() ?? string.Empty; if (!string.IsNullOrWhiteSpace(progress)) return progress; return "0/0 (0%)"; } private TimeSpan BuildRunDuration() { DateTime startedAtUtc = _runStartedAtUtc; if (startedAtUtc == default) return TimeSpan.Zero; TimeSpan duration = DateTime.UtcNow - startedAtUtc; if (duration < TimeSpan.Zero) duration = TimeSpan.Zero; return duration; } private static async Task CleanupStaleProductsAsync( DateTime cycleStartedAt, CancellationToken cancellationToken) { WorkerCore.UpdateActivity("Cleaning stale product fields"); WorkerCore.UpdateStatus(); var cleanupResult = await WorkerCore.ExecuteDatabaseSingleWithRetryAsync( ct => ProductWebInfoRecord.ClearOlderThan(cycleStartedAt, ct), OperationStaleRecordCleanup, cancellationToken) .ConfigureAwait(false); if (!cleanupResult.Success) { string error = cleanupResult.ErrorMessage ?? "Unknown cleanup error."; throw new WebSyncOperationException( OperationStaleRecordCleanup, $"Cleanup stale products failed. Threshold={cycleStartedAt:yyyy-MM-dd HH:mm:ss}. Error={error}", new InvalidOperationException(error)); } int clearedCount = cleanupResult.Item; WorkerCore.LogInfo( $"Cleanup completed. Cleared stale product records: {clearedCount}. Threshold={cycleStartedAt:yyyy-MM-dd HH:mm:ss}.", OperationStaleRecordCleanup); } private async Task DownloadCollectionDataAsync( RuntimeSettings settings, CancellationToken cancellationToken) { WorkerCore.UpdateActivity("Loading collection pages"); WorkerCore.UpdateStatus(); await WaitIfPausedAsync(cancellationToken).ConfigureAwait(false); var snapshot = new WebSyncCollectionSnapshot(); var links = new HashSet(StringComparer.OrdinalIgnoreCase); await WaitIfPausedAsync(cancellationToken).ConfigureAwait(false); var trending = await _collectionLoader.DownloadCollectionAsync( settings.TrendingUrl, settings.RetryCount, settings.DownloadTimeoutSeconds, settings.RetryDelaySeconds, OperationTrendingCollectionScan, _runReport, WaitIfPausedAsync, cancellationToken).ConfigureAwait(false); AddCollectionData(snapshot.TrendingArticleNumbers, trending, links); await WaitIfPausedAsync(cancellationToken).ConfigureAwait(false); var nieuw = await _collectionLoader.DownloadCollectionAsync( settings.NieuwUrl, settings.RetryCount, settings.DownloadTimeoutSeconds, settings.RetryDelaySeconds, OperationNieuwCollectionScan, _runReport, WaitIfPausedAsync, cancellationToken).ConfigureAwait(false); AddCollectionData(snapshot.NieuwArticleNumbers, nieuw, links); await WaitIfPausedAsync(cancellationToken).ConfigureAwait(false); var acties = await _collectionLoader.DownloadCollectionAsync( settings.ActiesUrl, settings.RetryCount, settings.DownloadTimeoutSeconds, settings.RetryDelaySeconds, OperationActiesCollectionScan, _runReport, WaitIfPausedAsync, cancellationToken).ConfigureAwait(false); AddCollectionData(snapshot.ActiesArticleNumbers, acties, links); WorkerCore.LogInfo( $"Collection pages loaded. Trending={snapshot.TrendingArticleNumbers.Count}, Nieuw={snapshot.NieuwArticleNumbers.Count}, Acties={snapshot.ActiesArticleNumbers.Count}, AdditionalLinks={links.Count}.", OperationTrendingCollectionScan); return new CollectionImportData { Snapshot = snapshot, ProductUrls = links.ToList() }; } private ValueTask WaitIfPausedAsync(CancellationToken cancellationToken) { _resumeSignal.Wait(cancellationToken); return ValueTask.CompletedTask; } private static void AddCollectionData( HashSet targetArticleNumbers, WebSyncCollectionLoader.CollectionScanResult scanResult, HashSet targetLinks) { foreach (string articleNumber in scanResult.ArticleNumbers) { targetArticleNumbers.Add(articleNumber); } foreach (string productUrl in scanResult.ProductUrls) { targetLinks.Add(productUrl); } } private static IReadOnlyList MergeProductLinks( IReadOnlyList sitemapLinks, IReadOnlyCollection collectionLinks) { var merged = new List(sitemapLinks.Count + collectionLinks.Count); var seen = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (string link in sitemapLinks) { if (seen.Add(link)) merged.Add(link); } foreach (string link in collectionLinks) { if (seen.Add(link)) merged.Add(link); } return merged; } private async Task UpdateDictionaryValuesAsync(CancellationToken cancellationToken) { WorkerCore.UpdateActivity("Updating dictionary values"); WorkerCore.UpdateStatus(); IReadOnlyList snapshot = _runReport.BuildDictionaryValueSnapshot(); if (snapshot.Count == 0) { WorkerCore.LogInfo("Dictionary values update skipped. No values collected.", OperationDictionaryValuesUpdate); return; } foreach (WebSyncRunReport.DictionaryColumnValues columnValues in snapshot) { var result = await WorkerCore.ExecuteDatabaseSingleWithRetryAsync( ct => DictionaryValueRecord.ReplaceColumnValues( columnValues.TableName, columnValues.ColumnName, columnValues.Values, ct), OperationDictionaryValuesUpdate, cancellationToken) .ConfigureAwait(false); if (!result.Success) { string error = result.ErrorMessage ?? "Unknown dictionary update error."; throw new WebSyncOperationException( OperationDictionaryValuesUpdate, $"Dictionary values update failed for {columnValues.TableName}.{columnValues.ColumnName}. Error={error}", new InvalidOperationException(error)); } } WorkerCore.LogInfo( $"Dictionary values update completed. Columns={snapshot.Count}.", OperationDictionaryValuesUpdate); } private static void ConfigureWorkerParameters() { WorkerCore.Parameters.Clear(); WorkerCore.Parameters.Add(new WorkerParameter { Parametr = TargetImagesPathKey, Value = @"c:\sonex\wwwroot\products\", Title = "Target image root path", Description = "Root folder where downloaded product images are stored.", ValueType = WorkerSettingValueTypes.String }); WorkerCore.Parameters.Add(new WorkerParameter { Parametr = SitemapUrlKey, Value = "https://www.xenos.nl/storage/sitemap/default/sitemap.xml", Title = "Sitemap URL", Description = "URL of the sitemap used to discover product pages.", ValueType = WorkerSettingValueTypes.String }); WorkerCore.Parameters.Add(new WorkerParameter { Parametr = TrendingUrlKey, Value = "https://www.xenos.nl/trending", Title = "Trending URL", Description = "Base URL of the trending collection pages.", ValueType = WorkerSettingValueTypes.String }); WorkerCore.Parameters.Add(new WorkerParameter { Parametr = NieuwUrlKey, Value = "https://www.xenos.nl/nieuw", Title = "Nieuw URL", Description = "Base URL of the nieuw collection pages.", ValueType = WorkerSettingValueTypes.String }); WorkerCore.Parameters.Add(new WorkerParameter { Parametr = ActiesUrlKey, Value = "https://www.xenos.nl/acties", Title = "Acties URL", Description = "Base URL of the acties collection pages.", ValueType = WorkerSettingValueTypes.String }); WorkerCore.Parameters.Add(new WorkerParameter { Parametr = MaxParallelTasksKey, Value = "20", Title = "Maximum parallel tasks", Description = "Maximum number of product tasks processed in parallel.", ValueType = WorkerSettingValueTypes.Integer }); WorkerCore.Parameters.Add(new WorkerParameter { Parametr = RetryCountKey, Value = "2", Title = "Retry count", Description = "Number of retry attempts after a failed operation.", ValueType = WorkerSettingValueTypes.Integer }); WorkerCore.Parameters.Add(new WorkerParameter { Parametr = DownloadTimeoutSecondsKey, Value = "5", Title = "Download timeout (seconds)", Description = "Maximum duration of a single download request in seconds.", ValueType = WorkerSettingValueTypes.Integer }); WorkerCore.Parameters.Add(new WorkerParameter { Parametr = RetryDelaySecondsKey, Value = "120", Title = "Retry delay (seconds)", Description = "Delay between retry attempts in seconds.", ValueType = WorkerSettingValueTypes.Integer }); WorkerCore.Parameters.Add(new WorkerParameter { Parametr = UpdateImagesKey, Value = "true", Title = "Update images", Description = "If true, synchronize local image files and product_web_images records.", ValueType = WorkerSettingValueTypes.Boolean }); } private sealed class RuntimeSettings { public string TargetImagesPath { get; init; } = string.Empty; public string SitemapUrl { get; init; } = string.Empty; public string TrendingUrl { get; init; } = string.Empty; public string NieuwUrl { get; init; } = string.Empty; public string ActiesUrl { get; init; } = string.Empty; public int MaxParallelTasks { get; init; } public int RetryCount { get; init; } public int DownloadTimeoutSeconds { get; init; } public int RetryDelaySeconds { get; init; } public bool UpdateImages { get; init; } } private sealed class CollectionImportData { public WebSyncCollectionSnapshot Snapshot { get; init; } = new(); public IReadOnlyList ProductUrls { get; init; } = []; } }