Prometheus Error: "data does not end with # EOF" during backfill using 'promtool tsdb create-blocks-from'

Question

I am trying to backfill Prometheus metrics data from files using a Docker container. My setup was working fine until I reinstalled Docker. Now, I am encountering the following error:

An error occurred: Command '['docker', 'exec', 'prometheus', '/bin/sh', '-c', 'promtool tsdb create-blocks-from openmetrics /etc/prometheus/data/openmetrics_prometheus_1720519200.txt /prometheus']' returned non-zero exit status 1.
stderr: getting min and max timestamp: next: data does not end with # EOF

I am running a script to backfill Prometheus metrics data from JSON files, converting them to OpenMetrics format, and appending # EOF to the end of each file. Here is the relevant part of my script:

import os
import subprocess
import json

def change_to_openmetrics(data, output_file_path):
    with open(output_file_path, "a") as output_file:
        for result in data["result"]:
            metric_name = result["metric"]["__name__"]
            labels = ",".join(
                [
                    f'{key}="{value}"'
                    for key, value in result["metric"].items()
                    if key != "__name__"
                ]
            )
            for value in result["values"]:
                timestamp = int(value[0])
                metric_value = float(value[1])
                output_file.write(
                    f"{metric_name}{{{labels}}} {metric_value} {timestamp}\n"
                )
    with open(output_file_path, "a") as output_file:
        output_file.write("# EOF\n")

def backfill_prometheus(folder_path):
    try:
        prometheus_container_name = "prometheus"
        files_found = False

        result = subprocess.run(
            [
                "docker",
                "ps",
                "--filter",
                f"name=^{prometheus_container_name}$",
                "--format",
                "{{.Names}}",
            ],
            capture_output=True,
            text=True,
        )
        if prometheus_container_name not in result.stdout.strip().split("\n"):
            print("Prometheus container is not running.")
            return

        if os.path.exists("./prometheus/data"):
            for f in os.listdir("./prometheus/data"):
                file_to_delete = os.path.join("./prometheus/data", f)
                if os.path.isfile(file_to_delete):
                    os.remove(file_to_delete)
                else:
                    print(f"Skipping non-file item: {file_to_delete}")

        for filename in os.listdir(folder_path):
            if not filename.endswith(".txt") and not filename.startswith("prometheus"):
                continue

            files_found = True
            file_path = os.path.join(folder_path, filename)
            openmetrics_filename = f"openmetrics_{filename}"
            openmetrics_file_path = os.path.join("./prometheus/data", openmetrics_filename)

            with open(file_path, "r") as file:
                for line in file:
                    try:
                        prometheus_data = json.loads(line.strip())
                        change_to_openmetrics(prometheus_data, openmetrics_file_path)
                    except json.JSONDecodeError:
                        print(f"Skipping line in file {filename} as it is not valid JSON.")

            with open(openmetrics_file_path, "a") as output_file:
                output_file.write("# EOF\n")

            cmd = (
                f"docker exec prometheus /bin/sh -c "
                f"'promtool tsdb create-blocks-from openmetrics /etc/prometheus/data/{openmetrics_filename} /prometheus'"
            )
            print(f"Running command: {cmd}")

            result = subprocess.run(
                [
                    "docker",
                    "exec",
                    "prometheus",
                    "/bin/sh",
                    "-c",
                    f"promtool tsdb create-blocks-from openmetrics /etc/prometheus/data/{openmetrics_filename} /prometheus",
                ],
                capture_output=True,
                text=True,
            )

            print("stdout:", result.stdout)
            print("stderr:", result.stderr)

            if result.returncode != 0:
                raise subprocess.CalledProcessError(result.returncode, result.args, output=result.stdout, stderr=result.stderr)

            print(f"Backfilled data from file: {filename}")

        if not files_found:
            print("No Prometheus data files found in the folder.")
        else:
            print("Backfilling complete for all files in the folder.")

    except subprocess.CalledProcessError as e:
        print(f"An error occurred: {e}")
        print("stdout:", e.output)
        print("stderr:", e.stderr)

# Example usage
# backfill_prometheus('/path/to/your/folder')

I tried runnig the command in terminal and its working succesfully, so the issue is not with the file.