Log Pipeline Integrations

{
  "uploadId": "9f3c2a01",
  "originalFilename": "app-2026-06-01.log",
  "originalBytes": 104857600,
  "compressedBytes": 4823104,
  "codec": "log-files",
  "compressionTimeMs": 2840,
  "createdAt": "2026-06-02T10:14:00Z"
}

echo "$(date) hello from smallest.zip" > /tmp/test.log
curl -X POST "https://smallest.zip/api/log-files/upload?level=balanced" \
  -H "x-api-key: YOUR_API_KEY" \
  -F "file=@/tmp/test.log"

curl -fsSL https://smallest.zip/install/logs | sh
export SMALLEST_API_KEY="YOUR_API_KEY"

# Compress a file in place (writes app.log.sbz, removes original with --rm)
smallest-logs compress app.log

# Stream stdin (good for piping from journalctl, kafka, etc.)
journalctl --since yesterday | smallest-logs compress - --out yesterday.sbz

import base64, gzip, json, os, time, urllib.request, uuid
import boto3

API_KEY = os.environ["SMALLEST_API_KEY"]
BUCKET  = os.environ["ARCHIVE_BUCKET"]
PREFIX  = os.environ.get("ARCHIVE_PREFIX", "cloudwatch/")
ENDPOINT = "https://smallest.zip/api/log-files/upload?level=max"

s3 = boto3.client("s3")

def handler(event, _ctx):
    # CloudWatch subscription payload is base64-gzipped JSON
    payload = json.loads(gzip.decompress(base64.b64decode(event["awslogs"]["data"])))
    lines = "\n".join(e["message"] for e in payload["logEvents"]).encode()

    boundary = uuid.uuid4().hex
    body = (
        f"--{boundary}\r\n"
        f'Content-Disposition: form-data; name="file"; filename="batch.log"\r\n'
        f"Content-Type: text/plain\r\n\r\n"
    ).encode() + lines + f"\r\n--{boundary}--\r\n".encode()

    req = urllib.request.Request(ENDPOINT, data=body, method="POST", headers={
        "x-api-key": API_KEY,
        "Content-Type": f"multipart/form-data; boundary={boundary}",
    })
    with urllib.request.urlopen(req, timeout=60) as r:
        meta = json.loads(r.read())

    # Download the compressed blob and stash it in our own S3 bucket
    dl = urllib.request.Request(
        f"https://smallest.zip/api/files/{meta['uploadId']}/download",
        headers={"x-api-key": API_KEY})
    with urllib.request.urlopen(dl, timeout=60) as r:
        blob = r.read()

    key = f"{PREFIX}{payload['logGroup']}/{int(time.time())}-{meta['uploadId']}.sbz"
    s3.put_object(Bucket=BUCKET, Key=key, Body=blob,
                  Metadata={"original-bytes": str(meta["originalBytes"]),
                            "compressed-bytes": str(meta["compressedBytes"])})
    return {"ok": True, "s3": f"s3://{BUCKET}/{key}", "ratio": meta["originalBytes"]/max(1, meta["compressedBytes"])}

aws lambda add-permission \
  --function-name smallestzip-forwarder \
  --statement-id cwlogs-invoke \
  --action lambda:InvokeFunction \
  --principal logs.amazonaws.com

aws logs put-subscription-filter \
  --log-group-name /aws/lambda/my-app \
  --filter-name smallestzip \
  --filter-pattern "" \
  --destination-arn arn:aws:lambda:us-east-1:123456789012:function:smallestzip-forwarder

{
  "Version": "2012-10-17",
  "Statement": [
    { "Effect": "Allow",
      "Action": ["logs:CreateLogStream", "logs:PutLogEvents"],
      "Resource": "arn:aws:logs:*:*:*" },
    { "Effect": "Allow",
      "Action": "s3:PutObject",
      "Resource": "arn:aws:s3:::my-log-archive/cloudwatch/*" }
  ]
}

# Trigger a log event, then within ~60s confirm a new .sbz landed in S3:
aws s3 ls s3://my-log-archive/cloudwatch/ --recursive | tail
# And confirm round-trip integrity:
aws s3 cp s3://my-log-archive/cloudwatch/<LATEST>.sbz /tmp/last.sbz
smallest-logs decompress /tmp/last.sbz --out - | head

[httpout]
httpEventCollectorToken = YOUR_API_KEY
uri = https://smallest.zip/api/log-files/upload?level=balanced
batchSize = 65536
batchTimeout = 30
sslVerifyServerCert = true

[tcpout]
defaultGroup = no_indexers

#!/usr/bin/env bash
set -euo pipefail
DAY=$(date -u -d 'yesterday' +%F)
OUT=/var/tmp/splunk-$DAY.log

/opt/splunk/bin/splunk search \
  "search index=* earliest=-1d@d latest=@d | table _raw" \
  -auth admin:$SPLUNK_PASS -output rawxml -maxout 0 \
  | sed -n 's:.*<text>\(.*\)</text>.*:\1:p' > "$OUT"

curl -fsS -X POST "https://smallest.zip/api/log-files/upload?level=max" \
  -H "x-api-key: $SMALLEST_API_KEY" \
  -F "file=@$OUT" | tee /var/log/smallest-splunk.log
rm "$OUT"

curl -s "https://smallest.zip/api/files?codec=log-files&limit=5" \
  -H "x-api-key: YOUR_API_KEY" | jq '.[] | {uploadId, originalBytes, compressedBytes}'

import gzip, json, os, uuid, urllib.request, boto3

API_KEY  = os.environ["SMALLEST_API_KEY"]
ENDPOINT = "https://smallest.zip/api/log-files/upload?level=max"
s3 = boto3.client("s3")

def handler(event, _):
    for rec in event["Records"]:
        b, k = rec["s3"]["bucket"]["name"], rec["s3"]["object"]["key"]
        if not k.startswith("raw/"): continue

        body = s3.get_object(Bucket=b, Key=k)["Body"].read()
        plain = gzip.decompress(body)

        boundary = uuid.uuid4().hex
        mp = (f"--{boundary}\r\n"
              f'Content-Disposition: form-data; name="file"; filename="dd.log"\r\n'
              f"Content-Type: text/plain\r\n\r\n").encode() + plain + \
             f"\r\n--{boundary}--\r\n".encode()

        req = urllib.request.Request(ENDPOINT, data=mp, method="POST", headers={
            "x-api-key": API_KEY,
            "Content-Type": f"multipart/form-data; boundary={boundary}"})
        meta = json.loads(urllib.request.urlopen(req, timeout=120).read())

        dl = urllib.request.Request(
            f"https://smallest.zip/api/files/{meta['uploadId']}/download",
            headers={"x-api-key": API_KEY})
        blob = urllib.request.urlopen(dl, timeout=120).read()

        new_key = k.replace("raw/", "sbz/").rsplit(".", 2)[0] + ".sbz"
        s3.put_object(Bucket=b, Key=new_key, Body=blob)
        s3.delete_object(Bucket=b, Key=k)   # drop the gzip original

aws s3 ls s3://my-dd-archive/sbz/ --recursive --human-readable | tail
# Pick the freshest .sbz and round-trip it locally:
aws s3 cp s3://my-dd-archive/sbz/<LATEST>.sbz - | smallest-logs decompress - --out - | head

<source>
  @type tail
  path /var/log/myapp/*.log
  pos_file /var/log/fluent/myapp.pos
  tag app.logs
  <parse>
    @type none
  </parse>
</source>

<match app.logs>
  @type http
  endpoint https://smallest.zip/api/log-files/upload?level=balanced
  open_timeout 10
  read_timeout 60
  <format>
    @type json
  </format>
  <buffer>
    @type file
    path /var/log/fluent/buf-smallest
    chunk_limit_size 8m
    flush_interval 30s
    retry_max_interval 60
  </buffer>
  <auth>
    method basic
  </auth>
  headers {"x-api-key":"YOUR_API_KEY"}
  json_array true
</match>

echo "fluentd test $(date)" | sudo tee -a /var/log/myapp/test.log
sleep 35  # wait one flush interval
curl -s "https://smallest.zip/api/files?codec=log-files&limit=1" \
  -H "x-api-key: YOUR_API_KEY" | jq

[sources.app_logs]
  type = "file"
  include = ["/var/log/myapp/*.log"]
  read_from = "end"

[transforms.batch_tag]
  type = "remap"
  inputs = ["app_logs"]
  source = '.batch = "smallest"'

[sinks.smallest_zip]
  type = "http"
  inputs = ["batch_tag"]
  uri = "https://smallest.zip/api/log-files/upload?level=balanced"
  method = "post"
  compression = "none"   # smallest.zip does its own compression
  encoding.codec = "text"
  framing.method = "newline_delimited"

  [sinks.smallest_zip.request.headers]
    x-api-key = "YOUR_API_KEY"
    content-type = "text/plain"

  [sinks.smallest_zip.batch]
    max_bytes = 8388608    # 8 MB per upload
    timeout_secs = 30

  [sinks.smallest_zip.buffer]
    type = "disk"
    max_size = 1073741824  # 1 GB on-disk overflow
    when_full = "block"

vector validate /etc/vector/vector.toml
sudo systemctl restart vector
# Tail the internal metrics endpoint:
curl -s http://localhost:8686/metrics | grep -E 'http_(sink|client)_.*smallest'

input {
  file {
    path => "/var/log/myapp/*.log"
    start_position => "end"
    sincedb_path => "/var/lib/logstash/sincedb-smallest"
  }
}

output {
  http {
    url => "https://smallest.zip/api/log-files/upload?level=balanced"
    http_method => "post"
    format => "message"
    content_type => "text/plain"
    headers => {
      "x-api-key" => "YOUR_API_KEY"
    }
    pool_max => 10
    socket_timeout => 60
    retry_failed => true
    retry_non_idempotent => true
  }
}

/usr/share/logstash/bin/logstash -t -f /etc/logstash/conf.d/smallest.conf
sudo systemctl restart logstash
echo "logstash test $(date)" | sudo tee -a /var/log/myapp/app.log
sleep 10
curl -s "https://smallest.zip/api/files?codec=log-files&limit=1" \
  -H "x-api-key: YOUR_API_KEY" | jq

SHELL=/bin/bash
SMALLEST_API_KEY=YOUR_API_KEY

# Ship yesterday's journal at 02:13 every night
13 2 * * * root journalctl --since "yesterday" --until "today" --no-pager \
  | /usr/local/bin/smallest-logs compress - \
      --out /var/log/archive/$(hostname)-$(date -u -d yesterday +\%F).sbz \
  && logger "smallest-logs: shipped $(date -u -d yesterday +\%F)"

#!/usr/bin/env bash
set -euo pipefail
: "${SMALLEST_API_KEY:?must be set}"

SRC=/var/log/syslog.1   # logrotate's previous-day file
[ -f "$SRC" ] || { echo "no rotated syslog yet"; exit 0; }

RESP=$(curl -fsS -X POST "https://smallest.zip/api/log-files/upload?level=max" \
        -H "x-api-key: $SMALLEST_API_KEY" \
        -F "file=@${SRC}")
echo "$RESP" | jq -r '"shipped \(.uploadId): \(.originalBytes) -> \(.compressedBytes) bytes"'

module(load="omprog")

template(name="RawMsg" type="string" string="%msg%\n")

action(type="omprog"
       binary="/usr/local/bin/smallest-logs compress - --rotate-out /var/log/archive/rsyslog --rotate-bytes 8388608"
       template="RawMsg")

# Dry-run the nightly cron job right now
SMALLEST_API_KEY=YOUR_API_KEY journalctl --since "1 hour ago" --no-pager \
  | smallest-logs compress - --out /tmp/last-hour.sbz
ls -lh /tmp/last-hour.sbz
smallest-logs decompress /tmp/last-hour.sbz --out - | tail