From f3ef283dcf2d5de67d6163e700b5de5b19ef6959 Mon Sep 17 00:00:00 2001
From: Keshav Priyadarshi <git@keshav.space>
Date: Thu, 22 May 2025 01:27:04 +0530
Subject: [PATCH 1/5] Staging migrations

Signed-off-by: Keshav Priyadarshi <git@keshav.space>
---
 .../0092_pipelineschedule_pipelinerun.py      | 17 +----------
 ...0093_pipelineschedule_execution_timeout.py | 30 +++++++++++++++++++
 2 files changed, 31 insertions(+), 16 deletions(-)
 create mode 100644 vulnerabilities/migrations/0093_pipelineschedule_execution_timeout.py

diff --git a/vulnerabilities/migrations/0092_pipelineschedule_pipelinerun.py b/vulnerabilities/migrations/0092_pipelineschedule_pipelinerun.py
index 852198416..e029596d8 100644
--- a/vulnerabilities/migrations/0092_pipelineschedule_pipelinerun.py
+++ b/vulnerabilities/migrations/0092_pipelineschedule_pipelinerun.py
@@ -1,4 +1,4 @@
-# Generated by Django 4.2.20 on 2025-05-21 19:32
+# Generated by Django 4.2.20 on 2025-05-12 17:04
 
 import django.core.validators
 from django.db import migrations, models
@@ -65,21 +65,6 @@ class Migration(migrations.Migration):
                         unique=True,
                     ),
                 ),
-                (
-                    "execution_timeout",
-                    models.PositiveSmallIntegerField(
-                        default=24,
-                        help_text="Number hours before pipeline execution is forcefully terminated.",
-                        validators=[
-                            django.core.validators.MinValueValidator(
-                                1, message="Pipeline timeout must be at least 1 hour."
-                            ),
-                            django.core.validators.MaxValueValidator(
-                                72, message="Pipeline timeout must be at most 72 hours."
-                            ),
-                        ],
-                    ),
-                ),
                 ("created_date", models.DateTimeField(auto_now_add=True, db_index=True)),
             ],
             options={
diff --git a/vulnerabilities/migrations/0093_pipelineschedule_execution_timeout.py b/vulnerabilities/migrations/0093_pipelineschedule_execution_timeout.py
new file mode 100644
index 000000000..8c8d1b59a
--- /dev/null
+++ b/vulnerabilities/migrations/0093_pipelineschedule_execution_timeout.py
@@ -0,0 +1,30 @@
+# Generated by Django 4.2.20 on 2025-05-21 19:56
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0092_pipelineschedule_pipelinerun"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="pipelineschedule",
+            name="execution_timeout",
+            field=models.PositiveSmallIntegerField(
+                default=24,
+                help_text="Number hours before pipeline execution is forcefully terminated.",
+                validators=[
+                    django.core.validators.MinValueValidator(
+                        1, message="Pipeline timeout must be at least 1 hour."
+                    ),
+                    django.core.validators.MaxValueValidator(
+                        72, message="Pipeline timeout must be at most 72 hours."
+                    ),
+                ],
+            ),
+        ),
+    ]

From 9578815257704c0d30395305a6bc3f393dbb44b8 Mon Sep 17 00:00:00 2001
From: Keshav Priyadarshi <git@keshav.space>
Date: Thu, 22 May 2025 15:55:19 +0530
Subject: [PATCH 2/5] Show timezone in logs and stats

Signed-off-by: Keshav Priyadarshi <git@keshav.space>
---
 vulnerabilities/pipelines/__init__.py          |  2 +-
 .../templates/pipeline_run_details.html        | 18 +++++++++++++++---
 .../templates/pipeline_run_list.html           |  4 ++--
 .../templates/pipeline_schedule_list.html      |  2 +-
 4 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py
index da200db48..64fa2608e 100644
--- a/vulnerabilities/pipelines/__init__.py
+++ b/vulnerabilities/pipelines/__init__.py
@@ -121,7 +121,7 @@ def execute(self):
     def log(self, message, level=logging.INFO):
         """Log the given `message` to the current module logger and execution_log."""
         now_local = datetime.now(timezone.utc).astimezone()
-        timestamp = now_local.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+        timestamp = now_local.strftime("%Y-%m-%d %T.%f %Z")
         message = f"{timestamp} {message}"
         module_logger.log(level, message)
         self.append_to_log(message)
diff --git a/vulnerabilities/templates/pipeline_run_details.html b/vulnerabilities/templates/pipeline_run_details.html
index 2953266da..83f971be4 100644
--- a/vulnerabilities/templates/pipeline_run_details.html
+++ b/vulnerabilities/templates/pipeline_run_details.html
@@ -77,15 +77,27 @@ <h1 class="title">{{ pipeline_name }} Run Log</h1>
                 </div>
                 <div class="column is-one-fifth-desktop is-one-quarter-tablet is-half-mobile">
                     <p class="is-size-7 has-text-weight-semibold">Start</p>
-                    <p class="has-text-grey is-size-7">{{ run.run_start_date|default:"N/A" }}</p>
+                    <p class="has-text-grey is-size-7">
+                        {% if run.run_start_date %}
+                            {{ run.run_start_date|date:"Y-m-d h:i a T" }}
+                        {% else %}
+                            N/A
+                        {% endif %}
+                    </p>
                 </div>
                 <div class="column is-one-fifth-desktop is-one-quarter-tablet is-half-mobile ">
                     <p class="is-size-7 has-text-weight-semibold">End</p>
-                    <p class="has-text-grey is-size-7">{{ run.run_end_date|default_if_none:"N/A" }}</p>
+                    <p class="has-text-grey is-size-7">
+                        {% if run.run_end_date %}
+                            {{ run.run_end_date|date:"Y-m-d h:i a T" }}
+                        {% else %}
+                            N/A
+                        {% endif %}
+                    </p>
                 </div>
                 <div class="column is-one-fifth-desktop is-one-quarter-tablet is-half-mobile">
                     <p class="is-size-7 has-text-weight-semibold">Created</p>
-                    <p class="has-text-grey is-size-7">{{ run.created_date }}</p>
+                    <p class="has-text-grey is-size-7">{{ run.created_date|date:"Y-m-d h:i a T" }}</p>
                 </div>
                 <div class="column is-one-fifth-desktop is-one-quarter-tablet is-half-mobile">
                     <p class="is-size-7 has-text-weight-semibold">Version</p>
diff --git a/vulnerabilities/templates/pipeline_run_list.html b/vulnerabilities/templates/pipeline_run_list.html
index ef87e986a..21c8007d0 100644
--- a/vulnerabilities/templates/pipeline_run_list.html
+++ b/vulnerabilities/templates/pipeline_run_list.html
@@ -77,14 +77,14 @@ <h1>{{ pipeline_name }} Runs</h1>
                   </div>
                   <div class="column is-one-fifth has-text-grey">
                     {% if run.run_start_date %}
-                      {{ run.run_start_date|date:"Y-m-d H:i" }}
+                      {{ run.run_start_date|date:"Y-m-d h:i a T" }}
                     {% else %}
                       N/A
                     {% endif %}
                   </div>
                   <div class="column is-one-fifth has-text-grey">
                     {% if run.run_end_date %}
-                      {{ run.run_end_date|date:"Y-m-d H:i" }}
+                      {{ run.run_end_date|date:"Y-m-d h:i a T" }}
                     {% else %}
                       N/A
                     {% endif %}
diff --git a/vulnerabilities/templates/pipeline_schedule_list.html b/vulnerabilities/templates/pipeline_schedule_list.html
index 6af521be2..12d117117 100644
--- a/vulnerabilities/templates/pipeline_schedule_list.html
+++ b/vulnerabilities/templates/pipeline_schedule_list.html
@@ -81,7 +81,7 @@ <h1>Pipeline Schedule</h1>
                                     <div class="column is-one-fifth has-text-grey"
                                         title="{{ schedule.latest_run_date }}">
                                         {% if schedule.latest_run_date %}
-                                            {{ schedule.latest_run_date|date:"Y-m-d H:i" }}
+                                            {{ schedule.latest_run_date|date:"Y-m-d h:i a T" }}
                                         {% else %}
                                             N/A
                                         {% endif %}

From e0537cea51de3f48f420c601a2c82ac1106c6c9e Mon Sep 17 00:00:00 2001
From: Keshav Priyadarshi <git@keshav.space>
Date: Thu, 22 May 2025 22:17:13 +0530
Subject: [PATCH 3/5] Add clientside pagination for bigger log snippets

Signed-off-by: Keshav Priyadarshi <git@keshav.space>
---
 .../templates/pipeline_run_details.html       | 62 +++++++++++++++++--
 1 file changed, 57 insertions(+), 5 deletions(-)

diff --git a/vulnerabilities/templates/pipeline_run_details.html b/vulnerabilities/templates/pipeline_run_details.html
index 83f971be4..28b3d7c9b 100644
--- a/vulnerabilities/templates/pipeline_run_details.html
+++ b/vulnerabilities/templates/pipeline_run_details.html
@@ -143,20 +143,37 @@ <h2 class="subtitle mb-2">Run Error</h2>
         <div class="box">
             <h2 class="subtitle mb-2">Log Output</h2>
             <div class="log-wrapper" style="position: relative;">
-                <button class="button is-medium is-light  copy-btn" id="copy-code"
+                <button class="button is-medium is-light copy-btn" id="copy-code"
                     onclick="copyCode('log-code', 'copy-code')">
                     <span class="icon is-medium">
                         <i class="fa fa-copy"></i>
                     </span>
                 </button>
-                <pre><code id="log-code" class="language-toml">{{ run.log }}</code></pre>
+
+                <pre><code id="log-code" class="language-toml" style="all: unset; display: flex; justify-content: center;">
+                    <i class="fa fa-spinner fa-spin fa-3x my-5 py-5"></i>
+                </code></pre>
+
+                <div class="has-text-centered mt-3">
+                    <button class="button is-link is-small mr-2" onclick="prevSnippet()">
+                        <i class="fa fa-arrow-left mr-1"></i>
+                        Prev
+                    </button>
+                    <span id="snippet-indicator">Snippet 1</span>
+                    <button class="button is-link is-small ml-2" onclick="nextSnippet()">
+                        Next
+                        <i class="fa fa-arrow-right ml-1"></i>
+                    </button>
+                </div>
             </div>
         </div>
+
         {% endif %}
 
 
-        <a href="{% url 'runs-list' pipeline_id=run.pipeline.pipeline_id %}" class="button is-info mt-4"><i class="fa fa-arrow-left mr-2"></i>Back to All
-            Runs</a>
+        <a href="{% url 'runs-list' pipeline_id=run.pipeline.pipeline_id %}" class="button is-info mt-4">
+            <i class="fa fa-arrow-left mr-2"></i>Back to AllRuns
+        </a>
     </div>
 </section>
 {% endblock %}
@@ -167,9 +184,44 @@ <h2 class="subtitle mb-2">Log Output</h2>
 <script src="{% static 'js/highlight-10.6.0.min.js' %}" crossorigin="anonymous"></script>
 <script>hljs.highlightAll();</script>
 
+<script>
+    const logText = `{{ run.log|escapejs }}`;
+    const lines = logText.split('\n');
+    const linesPerSnippet = 500;
+    const maxSnippetCount = Math.ceil(lines.length / linesPerSnippet);
+    let currentSnippet = 0;
+
+    function renderSnippet() {
+        const start = currentSnippet * linesPerSnippet;
+        const end = start + linesPerSnippet;
+        const pageLines = lines.slice(start, end).join('\n');
+        snippetIndicator = `Snippet ${currentSnippet + 1} of ${maxSnippetCount}`;
+        document.getElementById("log-code").removeAttribute("style");
+        document.getElementById('log-code').textContent = pageLines;
+        document.getElementById('snippet-indicator').textContent = snippetIndicator;
+    }
+
+    function nextSnippet() {
+        if ((currentSnippet + 1) < maxSnippetCount) {
+            currentSnippet++;
+            renderSnippet();
+        }
+    }
+
+    function prevSnippet() {
+        if (currentSnippet > 0) {
+            currentSnippet--;
+            renderSnippet();
+        }
+    }
+
+    renderSnippet();
+</script>
+
+
 <script>
     function copyCode(target, button) {
-        const code = document.getElementById(target).innerText;
+        const code = (target === "log-code") ? `{{ run.log|escapejs }}` : `{{ run.run_output|escapejs }}`;
         navigator.clipboard.writeText(code)
             .then(() => {
                 const btn = document.getElementById(button);

From a5383b54196ff497603d477ee90a3756e6393c9f Mon Sep 17 00:00:00 2001
From: Keshav Priyadarshi <git@keshav.space>
Date: Fri, 23 May 2025 17:51:09 +0530
Subject: [PATCH 4/5] Handle boundary condition in snippet navigation

Signed-off-by: Keshav Priyadarshi <git@keshav.space>
---
 .../templates/pipeline_run_details.html       | 54 ++++++++++---------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/vulnerabilities/templates/pipeline_run_details.html b/vulnerabilities/templates/pipeline_run_details.html
index 28b3d7c9b..8cb68c6af 100644
--- a/vulnerabilities/templates/pipeline_run_details.html
+++ b/vulnerabilities/templates/pipeline_run_details.html
@@ -5,6 +5,7 @@
 {% block title %}Run Log{% endblock %}
 
 {% block extrahead %}
+
 <style>
     pre {
         background-color: #282a36;
@@ -12,24 +13,13 @@
         padding: 1rem;
         border-radius: 4px;
         overflow-x: auto;
-        max-height: 500px;
+        height: 800px;
+        position: relative;
     }
 
     .column {
         word-break: break-word;
     }
-</style>
-
-<style>
-    pre {
-        background-color: #282a36;
-        color: #f8f8f2;
-        padding: 1.5rem 1rem 1rem 1rem;
-        border-radius: 4px;
-        overflow-x: auto;
-        max-height: 500px;
-        position: relative;
-    }
 
     .copy-btn {
         position: absolute;
@@ -113,7 +103,6 @@ <h1 class="title">{{ pipeline_name }} Run Log</h1>
                             <i class="fa fa-external-link fa_link_custom"></i>
                         </a>
                         {% endif %}
-
                     </p>
                 </div>
                 <div class="column is-one-fifth-desktop is-one-quarter-tablet is-half-mobile">
@@ -123,7 +112,6 @@ <h1 class="title">{{ pipeline_name }} Run Log</h1>
             </div>
         </div>
 
-
         {% if run.run_output|strip %}
         <div class="box">
             <h2 class="subtitle mb-2">Run Error</h2>
@@ -150,27 +138,26 @@ <h2 class="subtitle mb-2">Log Output</h2>
                     </span>
                 </button>
 
-                <pre><code id="log-code" class="language-toml" style="all: unset; display: flex; justify-content: center;">
-                    <i class="fa fa-spinner fa-spin fa-3x my-5 py-5"></i>
-                </code></pre>
+                <pre style="display: flex; justify-content: center; align-items: center;">
+                    <code id="log-code" class="language-toml"><i class="fa fa-spinner fa-spin fa-4x"></i></code>
+                </pre>
 
-                <div class="has-text-centered mt-3">
-                    <button class="button is-link is-small mr-2" onclick="prevSnippet()">
+                <div id="snippet-paginater" class="has-text-centered mt-3">
+                    <button id="prev-button" class="button is-link is-small mr-2" onclick="prevSnippet()">
                         <i class="fa fa-arrow-left mr-1"></i>
                         Prev
                     </button>
                     <span id="snippet-indicator">Snippet 1</span>
-                    <button class="button is-link is-small ml-2" onclick="nextSnippet()">
+                    <button id="next-button" class="button is-link is-small ml-2" onclick="nextSnippet()">
                         Next
                         <i class="fa fa-arrow-right ml-1"></i>
                     </button>
                 </div>
+
             </div>
         </div>
-
         {% endif %}
 
-
         <a href="{% url 'runs-list' pipeline_id=run.pipeline.pipeline_id %}" class="button is-info mt-4">
             <i class="fa fa-arrow-left mr-2"></i>Back to AllRuns
         </a>
@@ -182,7 +169,6 @@ <h2 class="subtitle mb-2">Log Output</h2>
 {% block scripts %}
 <link rel="stylesheet" href="{% static 'css/highlight-10.6.0.css' %}" crossorigin="anonymous">
 <script src="{% static 'js/highlight-10.6.0.min.js' %}" crossorigin="anonymous"></script>
-<script>hljs.highlightAll();</script>
 
 <script>
     const logText = `{{ run.log|escapejs }}`;
@@ -195,10 +181,26 @@ <h2 class="subtitle mb-2">Log Output</h2>
         const start = currentSnippet * linesPerSnippet;
         const end = start + linesPerSnippet;
         const pageLines = lines.slice(start, end).join('\n');
+
         snippetIndicator = `Snippet ${currentSnippet + 1} of ${maxSnippetCount}`;
-        document.getElementById("log-code").removeAttribute("style");
+        document.getElementById("log-code").parentElement.removeAttribute("style");
         document.getElementById('log-code').textContent = pageLines;
-        document.getElementById('snippet-indicator').textContent = snippetIndicator;
+        hljs.highlightAll();
+
+        if (maxSnippetCount > 1) {
+            const snippetIndicatorEl = document.getElementById('snippet-indicator');
+            const nextButton = document.getElementById('next-button');
+            const prevButton = document.getElementById('prev-button');
+
+            snippetIndicatorEl.textContent = snippetIndicator;
+
+            nextButton.disabled = (currentSnippet + 2 > maxSnippetCount);
+            prevButton.disabled = (currentSnippet - 1 < 0);
+        } else {
+            const paginator = document.getElementById('snippet-paginater');
+            paginator.innerHTML = '';
+        }
+
     }
 
     function nextSnippet() {

From 04db470c3cc8c94613ab0a6de1d3590b5314e011 Mon Sep 17 00:00:00 2001
From: Keshav Priyadarshi <git@keshav.space>
Date: Sat, 24 May 2025 21:56:03 +0530
Subject: [PATCH 5/5] Add setting to toggle live logging for pipelines

Signed-off-by: Keshav Priyadarshi <git@keshav.space>
---
 .../0094_pipelineschedule_live_logging.py     | 22 +++++++++++++++++++
 vulnerabilities/models.py                     | 10 +++++++++
 vulnerabilities/pipelines/__init__.py         |  9 +++++++-
 .../templates/pipeline_run_details.html       | 12 +++++++++-
 4 files changed, 51 insertions(+), 2 deletions(-)
 create mode 100644 vulnerabilities/migrations/0094_pipelineschedule_live_logging.py

diff --git a/vulnerabilities/migrations/0094_pipelineschedule_live_logging.py b/vulnerabilities/migrations/0094_pipelineschedule_live_logging.py
new file mode 100644
index 000000000..c00f6e973
--- /dev/null
+++ b/vulnerabilities/migrations/0094_pipelineschedule_live_logging.py
@@ -0,0 +1,22 @@
+# Generated by Django 4.2.20 on 2025-05-24 16:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0093_pipelineschedule_execution_timeout"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="pipelineschedule",
+            name="live_logging",
+            field=models.BooleanField(
+                db_index=True,
+                default=False,
+                help_text="When enabled logs will be streamed live during pipeline execution. For legacy importers and improvers, logs are always made available only after execution completes.",
+            ),
+        ),
+    ]
diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py
index 5cfdff611..bd9f8a1af 100644
--- a/vulnerabilities/models.py
+++ b/vulnerabilities/models.py
@@ -2129,6 +2129,16 @@ class PipelineSchedule(models.Model):
         ),
     )
 
+    live_logging = models.BooleanField(
+        null=False,
+        db_index=True,
+        default=False,
+        help_text=(
+            "When enabled logs will be streamed live during pipeline execution. "
+            "For legacy importers and improvers, logs are always made available only after execution completes."
+        ),
+    )
+
     run_interval = models.PositiveSmallIntegerField(
         validators=[
             MinValueValidator(1, message="Interval must be at least 1 day."),
diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py
index 64fa2608e..d41b05321 100644
--- a/vulnerabilities/pipelines/__init__.py
+++ b/vulnerabilities/pipelines/__init__.py
@@ -58,10 +58,15 @@ def __init__(
         self.current_step = ""
 
     def append_to_log(self, message):
-        if self.run:
+        if self.run and self.run.pipeline.live_logging:
             self.run.append_to_log(message)
         self.execution_log.append(message)
 
+    def update_final_run_log(self):
+        if self.run and not self.run.pipeline.live_logging:
+            final_log = "\n".join(self.execution_log)
+            self.run.append_to_log(final_log, is_multiline=True)
+
     def set_current_step(self, message):
         self.current_step = message
 
@@ -106,6 +111,7 @@ def execute(self):
                 self.on_failure()
                 on_failure_run_time = timer() - on_failure_start_time
                 self.log(f"Completed [on_failure] tasks in {humanize_time(on_failure_run_time)}")
+                self.update_final_run_log()
 
                 return 1, self.output_from_exception(exception)
 
@@ -115,6 +121,7 @@ def execute(self):
         self.set_current_step("")  # Reset the `current_step` field on completion
         pipeline_run_time = timer() - pipeline_start_time
         self.log(f"Pipeline completed in {humanize_time(pipeline_run_time)}")
+        self.update_final_run_log()
 
         return 0, ""
 
diff --git a/vulnerabilities/templates/pipeline_run_details.html b/vulnerabilities/templates/pipeline_run_details.html
index 8cb68c6af..ed75a74a1 100644
--- a/vulnerabilities/templates/pipeline_run_details.html
+++ b/vulnerabilities/templates/pipeline_run_details.html
@@ -112,6 +112,16 @@ <h1 class="title">{{ pipeline_name }} Run Log</h1>
             </div>
         </div>
 
+        {% if not run.run_end_date and not run.pipeline.live_logging %}
+            <div class="notification is-info is-light has-text-centered">
+                <p class="is-size-6 has-text-grey-dark">
+                    <i class="fa fa-exclamation-triangle mr-1"></i>
+                    Live logging is disabled for this pipeline.
+                    Logs will be available after the pipeline has finished running.
+                </p>
+            </div>
+        {% endif %}
+
         {% if run.run_output|strip %}
         <div class="box">
             <h2 class="subtitle mb-2">Run Error</h2>
@@ -159,7 +169,7 @@ <h2 class="subtitle mb-2">Log Output</h2>
         {% endif %}
 
         <a href="{% url 'runs-list' pipeline_id=run.pipeline.pipeline_id %}" class="button is-info mt-4">
-            <i class="fa fa-arrow-left mr-2"></i>Back to AllRuns
+            <i class="fa fa-arrow-left mr-2"></i>Back to All Runs
         </a>
     </div>
 </section>