diff --git a/vulnerabilities/migrations/0092_pipelineschedule_pipelinerun.py b/vulnerabilities/migrations/0092_pipelineschedule_pipelinerun.py
index 852198416..e029596d8 100644
--- a/vulnerabilities/migrations/0092_pipelineschedule_pipelinerun.py
+++ b/vulnerabilities/migrations/0092_pipelineschedule_pipelinerun.py
@@ -1,4 +1,4 @@
-# Generated by Django 4.2.20 on 2025-05-21 19:32
+# Generated by Django 4.2.20 on 2025-05-12 17:04
import django.core.validators
from django.db import migrations, models
@@ -65,21 +65,6 @@ class Migration(migrations.Migration):
unique=True,
),
),
- (
- "execution_timeout",
- models.PositiveSmallIntegerField(
- default=24,
- help_text="Number hours before pipeline execution is forcefully terminated.",
- validators=[
- django.core.validators.MinValueValidator(
- 1, message="Pipeline timeout must be at least 1 hour."
- ),
- django.core.validators.MaxValueValidator(
- 72, message="Pipeline timeout must be at most 72 hours."
- ),
- ],
- ),
- ),
("created_date", models.DateTimeField(auto_now_add=True, db_index=True)),
],
options={
diff --git a/vulnerabilities/migrations/0093_pipelineschedule_execution_timeout.py b/vulnerabilities/migrations/0093_pipelineschedule_execution_timeout.py
new file mode 100644
index 000000000..8c8d1b59a
--- /dev/null
+++ b/vulnerabilities/migrations/0093_pipelineschedule_execution_timeout.py
@@ -0,0 +1,30 @@
+# Generated by Django 4.2.20 on 2025-05-21 19:56
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("vulnerabilities", "0092_pipelineschedule_pipelinerun"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="pipelineschedule",
+ name="execution_timeout",
+ field=models.PositiveSmallIntegerField(
+ default=24,
+ help_text="Number hours before pipeline execution is forcefully terminated.",
+ validators=[
+ django.core.validators.MinValueValidator(
+ 1, message="Pipeline timeout must be at least 1 hour."
+ ),
+ django.core.validators.MaxValueValidator(
+ 72, message="Pipeline timeout must be at most 72 hours."
+ ),
+ ],
+ ),
+ ),
+ ]
diff --git a/vulnerabilities/migrations/0094_pipelineschedule_live_logging.py b/vulnerabilities/migrations/0094_pipelineschedule_live_logging.py
new file mode 100644
index 000000000..c00f6e973
--- /dev/null
+++ b/vulnerabilities/migrations/0094_pipelineschedule_live_logging.py
@@ -0,0 +1,22 @@
+# Generated by Django 4.2.20 on 2025-05-24 16:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("vulnerabilities", "0093_pipelineschedule_execution_timeout"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="pipelineschedule",
+ name="live_logging",
+ field=models.BooleanField(
+ db_index=True,
+ default=False,
+ help_text="When enabled logs will be streamed live during pipeline execution. For legacy importers and improvers, logs are always made available only after execution completes.",
+ ),
+ ),
+ ]
diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py
index 5cfdff611..bd9f8a1af 100644
--- a/vulnerabilities/models.py
+++ b/vulnerabilities/models.py
@@ -2129,6 +2129,16 @@ class PipelineSchedule(models.Model):
),
)
+ live_logging = models.BooleanField(
+ null=False,
+ db_index=True,
+ default=False,
+ help_text=(
+ "When enabled logs will be streamed live during pipeline execution. "
+ "For legacy importers and improvers, logs are always made available only after execution completes."
+ ),
+ )
+
run_interval = models.PositiveSmallIntegerField(
validators=[
MinValueValidator(1, message="Interval must be at least 1 day."),
diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py
index da200db48..d41b05321 100644
--- a/vulnerabilities/pipelines/__init__.py
+++ b/vulnerabilities/pipelines/__init__.py
@@ -58,10 +58,15 @@ def __init__(
self.current_step = ""
def append_to_log(self, message):
- if self.run:
+ if self.run and self.run.pipeline.live_logging:
self.run.append_to_log(message)
self.execution_log.append(message)
+ def update_final_run_log(self):
+ if self.run and not self.run.pipeline.live_logging:
+ final_log = "\n".join(self.execution_log)
+ self.run.append_to_log(final_log, is_multiline=True)
+
def set_current_step(self, message):
self.current_step = message
@@ -106,6 +111,7 @@ def execute(self):
self.on_failure()
on_failure_run_time = timer() - on_failure_start_time
self.log(f"Completed [on_failure] tasks in {humanize_time(on_failure_run_time)}")
+ self.update_final_run_log()
return 1, self.output_from_exception(exception)
@@ -115,13 +121,14 @@ def execute(self):
self.set_current_step("") # Reset the `current_step` field on completion
pipeline_run_time = timer() - pipeline_start_time
self.log(f"Pipeline completed in {humanize_time(pipeline_run_time)}")
+ self.update_final_run_log()
return 0, ""
def log(self, message, level=logging.INFO):
"""Log the given `message` to the current module logger and execution_log."""
now_local = datetime.now(timezone.utc).astimezone()
- timestamp = now_local.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+ timestamp = now_local.strftime("%Y-%m-%d %T.%f %Z")
message = f"{timestamp} {message}"
module_logger.log(level, message)
self.append_to_log(message)
diff --git a/vulnerabilities/templates/pipeline_run_details.html b/vulnerabilities/templates/pipeline_run_details.html
index 2953266da..ed75a74a1 100644
--- a/vulnerabilities/templates/pipeline_run_details.html
+++ b/vulnerabilities/templates/pipeline_run_details.html
@@ -5,6 +5,7 @@
{% block title %}Run Log{% endblock %}
{% block extrahead %}
+
-
-