556 lines
37 KiB
HTML
556 lines
37 KiB
HTML
|
||
|
||
<!DOCTYPE html>
|
||
<html class="writer-html5" lang="en" data-content_root="./">
|
||
<head>
|
||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
<meta content="tzst Performance Guide - Compression level optimization, performance tips, and comparison with other archive tools" name="description" />
|
||
<meta content="tzst performance, compression benchmarks, tar gzip comparison, archive performance optimization" name="keywords" />
|
||
<meta content="tzst Performance Guide" name="og:title" />
|
||
<meta content="Performance optimization tips and comparison with other archive tools for tzst" name="og:description" />
|
||
<meta content="tzst Performance Guide" name="twitter:title" />
|
||
<meta content="Performance optimization tips and comparison with other archive tools for tzst" name="twitter:description" />
|
||
<meta content="website" name="og:type" />
|
||
<meta content="https://tzst.xi-xu.me/_static/tzst-square-logo.png" name="og:image" />
|
||
<meta content="https://tzst.xi-xu.me/" name="og:url" />
|
||
<meta content="summary_large_image" name="twitter:card" />
|
||
<meta content="https://tzst.xi-xu.me/_static/tzst-square-logo.png" name="twitter:image" />
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>Performance Guide — tzst 1.3.3 Documentation</title>
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||
|
||
|
||
<link rel="shortcut icon" href="_static/favicon.ico"/>
|
||
<link rel="canonical" href="https://tzst.xi-xu.me/performance.html" />
|
||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||
<script src="_static/documentation_options.js?v=3b3401d5"></script>
|
||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||
<script src="_static/js/theme.js"></script>
|
||
<link rel="search" type="application/opensearchdescription+xml"
|
||
title="Search within tzst 1.3.3 Documentation"
|
||
href="_static/opensearch.xml"/>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="Examples" href="examples.html" />
|
||
<link rel="prev" title="Quick Start Guide" href="quickstart.html" />
|
||
|
||
<!-- Additional SEO and social meta tags -->
|
||
<meta name="application-name" content="tzst" />
|
||
<meta name="generator" content="Sphinx 9.1.0" />
|
||
<meta name="rating" content="General" />
|
||
<meta name="revisit-after" content="7 days" />
|
||
|
||
<!-- Schema.org markup for search engines -->
|
||
<script type="application/ld+json">
|
||
{
|
||
"@context": "https://schema.org",
|
||
"@type": "SoftwareApplication",
|
||
"name": "tzst",
|
||
"description": "A Python library for creating and extracting tar.zst archives with high performance and comprehensive features",
|
||
"applicationCategory": "DeveloperApplication",
|
||
"operatingSystem": "Cross-platform",
|
||
"programmingLanguage": "Python",
|
||
"license": "https://opensource.org/licenses/BSD-3-Clause",
|
||
"url": "https://tzst.xi-xu.me/",
|
||
"downloadUrl": "https://pypi.org/project/tzst/",
|
||
"codeRepository": "https://github.com/xixu-me/tzst",
|
||
"softwareVersion": "1.3.3",
|
||
"author": {
|
||
"@type": "Person",
|
||
"name": "Xi Xu",
|
||
"url": "https://xi-xu.me"
|
||
},
|
||
"offers": {
|
||
"@type": "Offer",
|
||
"price": "0",
|
||
"priceCurrency": "USD"
|
||
},
|
||
"aggregateRating": {
|
||
"@type": "AggregateRating",
|
||
"ratingValue": "5",
|
||
"reviewCount": "1"
|
||
},
|
||
"keywords": "tzst, tar, zstandard, compression, archive, python, extraction, backup"
|
||
}
|
||
</script>
|
||
|
||
<!-- Breadcrumb Schema -->
|
||
|
||
<script type="application/ld+json">
|
||
{
|
||
"@context": "https://schema.org",
|
||
"@type": "BreadcrumbList",
|
||
"itemListElement": [
|
||
{
|
||
"@type": "ListItem",
|
||
"position": 1,
|
||
"name": "Home",
|
||
"item": "https://tzst.xi-xu.me/"
|
||
},
|
||
{
|
||
"@type": "ListItem",
|
||
"position": 2,
|
||
"name": "Performance Guide",
|
||
"item": "https://tzst.xi-xu.me/performance.html"
|
||
}
|
||
]
|
||
}
|
||
</script>
|
||
|
||
|
||
<!-- Article/TechArticle Schema for documentation pages -->
|
||
|
||
<script type="application/ld+json">
|
||
{
|
||
"@context": "https://schema.org",
|
||
"@type": "TechArticle",
|
||
"headline": "Performance Guide",
|
||
"description": "",
|
||
"author": {
|
||
"@type": "Person",
|
||
"name": "Xi Xu",
|
||
"url": "https://xi-xu.me"
|
||
},
|
||
"publisher": {
|
||
"@type": "Person",
|
||
"name": "Xi Xu"
|
||
},
|
||
"datePublished": "2025-01-01",
|
||
"dateModified": "2025-01-12",
|
||
"url": "https://tzst.xi-xu.me/performance.html",
|
||
"inLanguage": "en-US",
|
||
"about": {
|
||
"@type": "SoftwareApplication",
|
||
"name": "tzst"
|
||
}
|
||
}
|
||
</script>
|
||
|
||
|
||
<!-- FAQ Schema for pages with common questions -->
|
||
|
||
|
||
<!-- HowTo Schema for examples page -->
|
||
|
||
|
||
<!-- Canonical URL for better SEO -->
|
||
|
||
<link rel="canonical" href="https://tzst.xi-xu.me/performance.html" />
|
||
|
||
|
||
<!-- Preconnect to external domains for performance -->
|
||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||
<link rel="preconnect" href="https://cdnjs.cloudflare.com" />
|
||
<link rel="dns-prefetch" href="https://pypi.org" />
|
||
<link rel="dns-prefetch" href="https://github.com" />
|
||
|
||
</head>
|
||
|
||
<body class="wy-body-for-nav">
|
||
<div class="wy-grid-for-nav">
|
||
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||
<div class="wy-side-scroll">
|
||
<div class="wy-side-nav-search" style="background: #2980B9" >
|
||
|
||
|
||
|
||
<a href="index.html" class="icon icon-home">
|
||
tzst
|
||
<img src="_static/tzst-logo.png" class="logo" alt="Logo"/>
|
||
</a>
|
||
<div role="search">
|
||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||
<input type="hidden" name="check_keywords" value="yes" />
|
||
<input type="hidden" name="area" value="default" />
|
||
</form>
|
||
</div>
|
||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||
<ul class="current">
|
||
<li class="toctree-l1"><a class="reference internal" href="quickstart.html">Quick Start Guide</a></li>
|
||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Performance Guide</a><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="#performance-tips">Performance Tips</a><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="#compression-levels">1. Compression Levels</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#streaming">2. Streaming</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#batch-operations">3. Batch Operations</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#file-type-considerations">4. File Type Considerations</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#comparison-with-other-tools">Comparison with Other Tools</a><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="#vs-tar-gzip">vs tar + gzip</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#vs-tar-xz">vs tar + xz</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#vs-zip">vs zip</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#benchmarking-examples">Benchmarking Examples</a><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="#compression-level-benchmark">Compression Level Benchmark</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#memory-usage-comparison">Memory Usage Comparison</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#best-practices">Best Practices</a><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="#for-development">For Development</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#for-backups">For Backups</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#for-distribution">For Distribution</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#for-archival-storage">For Archival Storage</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#hardware-considerations">Hardware Considerations</a><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="#cpu-usage">CPU Usage</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#memory-usage">Memory Usage</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#storage">Storage</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#integration-with-build-systems">Integration with Build Systems</a><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="#makefile-example">Makefile Example</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#github-actions-example">GitHub Actions Example</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="examples.html">Examples</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="api/index.html">API Reference</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="development.html">Development Guide</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="genindex.html">Index</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
</nav>
|
||
|
||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" style="background: #2980B9" >
|
||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||
<a href="index.html">tzst</a>
|
||
</nav>
|
||
|
||
<div class="wy-nav-content">
|
||
<div class="rst-content">
|
||
<div role="navigation" aria-label="Page navigation">
|
||
<ul class="wy-breadcrumbs">
|
||
<li><a href="index.html" class="icon icon-home" aria-label="Home"></a></li>
|
||
<li class="breadcrumb-item active">Performance Guide</li>
|
||
<li class="wy-breadcrumbs-aside">
|
||
<a href="https://github.com/xixu-me/tzst/blob/main/docs/performance.md" class="fa fa-github"> Edit on GitHub</a>
|
||
</li>
|
||
</ul>
|
||
<hr/>
|
||
</div>
|
||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||
<div itemprop="articleBody">
|
||
|
||
<section id="performance-guide">
|
||
<h1>Performance Guide<a class="headerlink" href="#performance-guide" title="Link to this heading"></a></h1>
|
||
<p>This guide covers performance optimization techniques and provides detailed comparisons with other archive tools.</p>
|
||
<section id="performance-tips">
|
||
<h2>Performance Tips<a class="headerlink" href="#performance-tips" title="Link to this heading"></a></h2>
|
||
<section id="compression-levels">
|
||
<h3>1. Compression Levels<a class="headerlink" href="#compression-levels" title="Link to this heading"></a></h3>
|
||
<p>Choose the right compression level for your use case:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Level 1-3</strong>: Fast compression, larger files (good for temporary archives or real-time processing)</p></li>
|
||
<li><p><strong>Level 3</strong> (default): Optimal balance for most use cases</p></li>
|
||
<li><p><strong>Level 6-9</strong>: Higher compression, moderate speed (good for regular backups)</p></li>
|
||
<li><p><strong>Level 15-22</strong>: Maximum compression, slower (for long-term storage or bandwidth-limited scenarios)</p></li>
|
||
</ul>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">tzst</span><span class="w"> </span><span class="kn">import</span> <span class="n">create_archive</span>
|
||
|
||
<span class="c1"># For temporary files or frequent operations</span>
|
||
<span class="n">create_archive</span><span class="p">(</span><span class="s2">"temp.tzst"</span><span class="p">,</span> <span class="n">files</span><span class="p">,</span> <span class="n">compression_level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Balanced default (recommended)</span>
|
||
<span class="n">create_archive</span><span class="p">(</span><span class="s2">"backup.tzst"</span><span class="p">,</span> <span class="n">files</span><span class="p">,</span> <span class="n">compression_level</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Long-term storage</span>
|
||
<span class="n">create_archive</span><span class="p">(</span><span class="s2">"archive.tzst"</span><span class="p">,</span> <span class="n">files</span><span class="p">,</span> <span class="n">compression_level</span><span class="o">=</span><span class="mi">9</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Maximum compression for critical space savings</span>
|
||
<span class="n">create_archive</span><span class="p">(</span><span class="s2">"minimal.tzst"</span><span class="p">,</span> <span class="n">files</span><span class="p">,</span> <span class="n">compression_level</span><span class="o">=</span><span class="mi">22</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="streaming">
|
||
<h3>2. Streaming<a class="headerlink" href="#streaming" title="Link to this heading"></a></h3>
|
||
<p>Use streaming mode for archives larger than 100MB:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">tzst</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_archive</span><span class="p">,</span> <span class="n">list_archive</span><span class="p">,</span> <span class="n">test_archive</span>
|
||
|
||
<span class="c1"># Memory-efficient operations for large archives</span>
|
||
<span class="n">extract_archive</span><span class="p">(</span><span class="s2">"large-backup.tzst"</span><span class="p">,</span> <span class="s2">"restore/"</span><span class="p">,</span> <span class="n">streaming</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
<span class="n">contents</span> <span class="o">=</span> <span class="n">list_archive</span><span class="p">(</span><span class="s2">"large-backup.tzst"</span><span class="p">,</span> <span class="n">streaming</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
<span class="n">is_valid</span> <span class="o">=</span> <span class="n">test_archive</span><span class="p">(</span><span class="s2">"large-backup.tzst"</span><span class="p">,</span> <span class="n">streaming</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Streaming Benefits:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Significantly reduced memory usage</p></li>
|
||
<li><p>Better performance for large archives</p></li>
|
||
<li><p>Handles archives that don’t fit in memory</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="batch-operations">
|
||
<h3>3. Batch Operations<a class="headerlink" href="#batch-operations" title="Link to this heading"></a></h3>
|
||
<p>Add multiple files in a single session when possible:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">tzst</span><span class="w"> </span><span class="kn">import</span> <span class="n">TzstArchive</span>
|
||
|
||
<span class="c1"># Efficient: Single archive session</span>
|
||
<span class="k">with</span> <span class="n">TzstArchive</span><span class="p">(</span><span class="s2">"backup.tzst"</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">)</span> <span class="k">as</span> <span class="n">archive</span><span class="p">:</span>
|
||
<span class="n">archive</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="s2">"file1.txt"</span><span class="p">)</span>
|
||
<span class="n">archive</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="s2">"file2.txt"</span><span class="p">)</span>
|
||
<span class="n">archive</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="s2">"directory/"</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Less efficient: Multiple separate operations</span>
|
||
<span class="n">create_archive</span><span class="p">(</span><span class="s2">"backup1.tzst"</span><span class="p">,</span> <span class="p">[</span><span class="s2">"file1.txt"</span><span class="p">])</span>
|
||
<span class="n">create_archive</span><span class="p">(</span><span class="s2">"backup2.tzst"</span><span class="p">,</span> <span class="p">[</span><span class="s2">"file2.txt"</span><span class="p">])</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="file-type-considerations">
|
||
<h3>4. File Type Considerations<a class="headerlink" href="#file-type-considerations" title="Link to this heading"></a></h3>
|
||
<ul class="simple">
|
||
<li><p>Already compressed files (<code class="docutils literal notranslate"><span class="pre">.jpg</span></code>, <code class="docutils literal notranslate"><span class="pre">.png</span></code>, <code class="docutils literal notranslate"><span class="pre">.mp4</span></code>, <code class="docutils literal notranslate"><span class="pre">.pdf</span></code>) won’t compress much further</p></li>
|
||
<li><p>Text files, source code, and logs compress very well</p></li>
|
||
<li><p>Consider compression level based on your data types</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="comparison-with-other-tools">
|
||
<h2>Comparison with Other Tools<a class="headerlink" href="#comparison-with-other-tools" title="Link to this heading"></a></h2>
|
||
<section id="vs-tar-gzip">
|
||
<h3>vs tar + gzip<a class="headerlink" href="#vs-tar-gzip" title="Link to this heading"></a></h3>
|
||
<p><strong>tzst Advantages:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Better compression ratios</strong>: 10-40% smaller archives</p></li>
|
||
<li><p><strong>Faster decompression</strong>: 2-3x faster extraction</p></li>
|
||
<li><p><strong>Modern algorithm</strong>: Better handling of various file types</p></li>
|
||
<li><p><strong>Streaming support</strong>: Better memory efficiency</p></li>
|
||
</ul>
|
||
<p><strong>When to use tar + gzip:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Legacy system compatibility requirements</p></li>
|
||
<li><p>Very old systems without zstd support</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="vs-tar-xz">
|
||
<h3>vs tar + xz<a class="headerlink" href="#vs-tar-xz" title="Link to this heading"></a></h3>
|
||
<p><strong>tzst Advantages:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Significantly faster compression</strong>: 3-10x faster creation</p></li>
|
||
<li><p><strong>Faster decompression</strong>: 2-4x faster extraction</p></li>
|
||
<li><p><strong>Better speed/compression trade-off</strong>: Similar compression with much better speed</p></li>
|
||
<li><p><strong>More compression levels</strong>: Fine-grained control (22 levels vs 9)</p></li>
|
||
</ul>
|
||
<p><strong>When to use tar + xz:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Maximum compression is critical and time is not a factor</p></li>
|
||
<li><p>Systems that don’t support zstd</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="vs-zip">
|
||
<h3>vs zip<a class="headerlink" href="#vs-zip" title="Link to this heading"></a></h3>
|
||
<p><strong>tzst Advantages:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Better compression</strong>: 15-30% smaller archives</p></li>
|
||
<li><p><strong>Preserves Unix permissions and metadata</strong>: Full POSIX compatibility</p></li>
|
||
<li><p><strong>Better streaming support</strong>: Memory-efficient for large archives</p></li>
|
||
<li><p><strong>Better directory handling</strong>: Preserves directory structure and timestamps</p></li>
|
||
</ul>
|
||
<p><strong>When to use zip:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Cross-platform compatibility with very old systems</p></li>
|
||
<li><p>Individual file access without full extraction is required</p></li>
|
||
<li><p>Windows-centric environments with no command-line tools</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="benchmarking-examples">
|
||
<h2>Benchmarking Examples<a class="headerlink" href="#benchmarking-examples" title="Link to this heading"></a></h2>
|
||
<section id="compression-level-benchmark">
|
||
<h3>Compression Level Benchmark<a class="headerlink" href="#compression-level-benchmark" title="Link to this heading"></a></h3>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tzst</span><span class="w"> </span><span class="kn">import</span> <span class="n">create_archive</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">benchmark_compression_levels</span><span class="p">(</span><span class="n">files</span><span class="p">,</span> <span class="n">output_prefix</span><span class="o">=</span><span class="s2">"benchmark"</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""Compare different compression levels."""</span>
|
||
<span class="n">levels_to_test</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">9</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">22</span><span class="p">]</span>
|
||
|
||
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="k">for</span> <span class="n">level</span> <span class="ow">in</span> <span class="n">levels_to_test</span><span class="p">:</span>
|
||
<span class="n">output_file</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">output_prefix</span><span class="si">}</span><span class="s2">_level_</span><span class="si">{</span><span class="n">level</span><span class="si">}</span><span class="s2">.tzst"</span>
|
||
|
||
<span class="c1"># Measure compression time</span>
|
||
<span class="n">start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
|
||
<span class="n">create_archive</span><span class="p">(</span><span class="n">output_file</span><span class="p">,</span> <span class="n">files</span><span class="p">,</span> <span class="n">compression_level</span><span class="o">=</span><span class="n">level</span><span class="p">)</span>
|
||
<span class="n">compress_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">-</span> <span class="n">start_time</span>
|
||
|
||
<span class="c1"># Get file size</span>
|
||
<span class="n">file_size</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">output_file</span><span class="p">)</span><span class="o">.</span><span class="n">stat</span><span class="p">()</span><span class="o">.</span><span class="n">st_size</span>
|
||
|
||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
|
||
<span class="s1">'level'</span><span class="p">:</span> <span class="n">level</span><span class="p">,</span>
|
||
<span class="s1">'time'</span><span class="p">:</span> <span class="n">compress_time</span><span class="p">,</span>
|
||
<span class="s1">'size'</span><span class="p">:</span> <span class="n">file_size</span><span class="p">,</span>
|
||
<span class="s1">'size_mb'</span><span class="p">:</span> <span class="n">file_size</span> <span class="o">/</span> <span class="p">(</span><span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">)</span>
|
||
<span class="p">})</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Level </span><span class="si">{</span><span class="n">level</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">compress_time</span><span class="si">:</span><span class="s2">.2f</span><span class="si">}</span><span class="s2">s, </span><span class="si">{</span><span class="n">file_size</span><span class="o">/</span><span class="mi">1024</span><span class="o">/</span><span class="mi">1024</span><span class="si">:</span><span class="s2">.1f</span><span class="si">}</span><span class="s2"> MB"</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">results</span>
|
||
|
||
<span class="c1"># Example usage</span>
|
||
<span class="n">files</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"documents/"</span><span class="p">,</span> <span class="s2">"projects/"</span><span class="p">]</span>
|
||
<span class="n">results</span> <span class="o">=</span> <span class="n">benchmark_compression_levels</span><span class="p">(</span><span class="n">files</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="memory-usage-comparison">
|
||
<h3>Memory Usage Comparison<a class="headerlink" href="#memory-usage-comparison" title="Link to this heading"></a></h3>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">psutil</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tzst</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_archive</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">monitor_memory_usage</span><span class="p">(</span><span class="n">func</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""Monitor memory usage during function execution."""</span>
|
||
<span class="n">process</span> <span class="o">=</span> <span class="n">psutil</span><span class="o">.</span><span class="n">Process</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">getpid</span><span class="p">())</span>
|
||
<span class="n">initial_memory</span> <span class="o">=</span> <span class="n">process</span><span class="o">.</span><span class="n">memory_info</span><span class="p">()</span><span class="o">.</span><span class="n">rss</span> <span class="o">/</span> <span class="mi">1024</span> <span class="o">/</span> <span class="mi">1024</span> <span class="c1"># MB</span>
|
||
|
||
<span class="n">func</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
|
||
<span class="n">peak_memory</span> <span class="o">=</span> <span class="n">process</span><span class="o">.</span><span class="n">memory_info</span><span class="p">()</span><span class="o">.</span><span class="n">rss</span> <span class="o">/</span> <span class="mi">1024</span> <span class="o">/</span> <span class="mi">1024</span> <span class="c1"># MB</span>
|
||
<span class="k">return</span> <span class="n">peak_memory</span> <span class="o">-</span> <span class="n">initial_memory</span>
|
||
|
||
<span class="c1"># Compare streaming vs non-streaming extraction</span>
|
||
<span class="n">large_archive</span> <span class="o">=</span> <span class="s2">"large-dataset.tzst"</span>
|
||
|
||
<span class="n">memory_normal</span> <span class="o">=</span> <span class="n">monitor_memory_usage</span><span class="p">(</span><span class="n">extract_archive</span><span class="p">,</span> <span class="n">large_archive</span><span class="p">,</span> <span class="s2">"output1/"</span><span class="p">)</span>
|
||
<span class="n">memory_streaming</span> <span class="o">=</span> <span class="n">monitor_memory_usage</span><span class="p">(</span><span class="n">extract_archive</span><span class="p">,</span> <span class="n">large_archive</span><span class="p">,</span> <span class="s2">"output2/"</span><span class="p">,</span> <span class="n">streaming</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Normal extraction: </span><span class="si">{</span><span class="n">memory_normal</span><span class="si">:</span><span class="s2">.1f</span><span class="si">}</span><span class="s2"> MB"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Streaming extraction: </span><span class="si">{</span><span class="n">memory_streaming</span><span class="si">:</span><span class="s2">.1f</span><span class="si">}</span><span class="s2"> MB"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Memory savings: </span><span class="si">{</span><span class="n">memory_normal</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">memory_streaming</span><span class="si">:</span><span class="s2">.1f</span><span class="si">}</span><span class="s2"> MB"</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="best-practices">
|
||
<h2>Best Practices<a class="headerlink" href="#best-practices" title="Link to this heading"></a></h2>
|
||
<section id="for-development">
|
||
<h3>For Development<a class="headerlink" href="#for-development" title="Link to this heading"></a></h3>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Fast compression for frequent builds</span>
|
||
<span class="n">create_archive</span><span class="p">(</span><span class="s2">"build-artifacts.tzst"</span><span class="p">,</span> <span class="p">[</span><span class="s2">"build/"</span><span class="p">],</span> <span class="n">compression_level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="for-backups">
|
||
<h3>For Backups<a class="headerlink" href="#for-backups" title="Link to this heading"></a></h3>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Balanced compression for regular backups</span>
|
||
<span class="n">create_archive</span><span class="p">(</span><span class="s2">"daily-backup.tzst"</span><span class="p">,</span> <span class="p">[</span><span class="s2">"data/"</span><span class="p">],</span> <span class="n">compression_level</span><span class="o">=</span><span class="mi">6</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="for-distribution">
|
||
<h3>For Distribution<a class="headerlink" href="#for-distribution" title="Link to this heading"></a></h3>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Higher compression for software distribution</span>
|
||
<span class="n">create_archive</span><span class="p">(</span><span class="s2">"software-package.tzst"</span><span class="p">,</span> <span class="p">[</span><span class="s2">"app/"</span><span class="p">],</span> <span class="n">compression_level</span><span class="o">=</span><span class="mi">9</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="for-archival-storage">
|
||
<h3>For Archival Storage<a class="headerlink" href="#for-archival-storage" title="Link to this heading"></a></h3>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Maximum compression for long-term storage</span>
|
||
<span class="n">create_archive</span><span class="p">(</span><span class="s2">"archive-2024.tzst"</span><span class="p">,</span> <span class="p">[</span><span class="s2">"historical-data/"</span><span class="p">],</span> <span class="n">compression_level</span><span class="o">=</span><span class="mi">22</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="hardware-considerations">
|
||
<h2>Hardware Considerations<a class="headerlink" href="#hardware-considerations" title="Link to this heading"></a></h2>
|
||
<section id="cpu-usage">
|
||
<h3>CPU Usage<a class="headerlink" href="#cpu-usage" title="Link to this heading"></a></h3>
|
||
<ul class="simple">
|
||
<li><p>Higher compression levels use more CPU but for shorter time periods</p></li>
|
||
<li><p>Modern multi-core systems handle zstd compression very efficiently</p></li>
|
||
<li><p>Consider system load when choosing compression levels</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="memory-usage">
|
||
<h3>Memory Usage<a class="headerlink" href="#memory-usage" title="Link to this heading"></a></h3>
|
||
<ul class="simple">
|
||
<li><p>Streaming mode: ~16-32 MB memory usage regardless of archive size</p></li>
|
||
<li><p>Normal mode: Memory usage proportional to archive size</p></li>
|
||
<li><p>Use streaming for archives >100 MB or on memory-constrained systems</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="storage">
|
||
<h3>Storage<a class="headerlink" href="#storage" title="Link to this heading"></a></h3>
|
||
<ul class="simple">
|
||
<li><p>SSDs benefit from higher compression (less I/O)</p></li>
|
||
<li><p>HDDs may prefer lower compression levels (CPU vs I/O trade-off)</p></li>
|
||
<li><p>Network storage benefits from higher compression (bandwidth savings)</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="integration-with-build-systems">
|
||
<h2>Integration with Build Systems<a class="headerlink" href="#integration-with-build-systems" title="Link to this heading"></a></h2>
|
||
<section id="makefile-example">
|
||
<h3>Makefile Example<a class="headerlink" href="#makefile-example" title="Link to this heading"></a></h3>
|
||
<div class="highlight-makefile notranslate"><div class="highlight"><pre><span></span><span class="c"># Fast compression for development</span>
|
||
<span class="nf">build-dev</span><span class="o">:</span><span class="w"> </span>
|
||
<span class="w"> </span>tzst<span class="w"> </span>a<span class="w"> </span>build-dev.tzst<span class="w"> </span>build/<span class="w"> </span>-l<span class="w"> </span><span class="m">1</span>
|
||
|
||
<span class="c"># Production compression</span>
|
||
<span class="nf">build-prod</span><span class="o">:</span>
|
||
<span class="w"> </span>tzst<span class="w"> </span>a<span class="w"> </span>build-prod.tzst<span class="w"> </span>build/<span class="w"> </span>-l<span class="w"> </span><span class="m">9</span>
|
||
|
||
<span class="c"># CI/CD artifacts</span>
|
||
<span class="nf">artifacts</span><span class="o">:</span>
|
||
<span class="w"> </span>tzst<span class="w"> </span>a<span class="w"> </span>artifacts.tzst<span class="w"> </span>dist/<span class="w"> </span>logs/<span class="w"> </span>-l<span class="w"> </span><span class="m">6</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="github-actions-example">
|
||
<h3>GitHub Actions Example<a class="headerlink" href="#github-actions-example" title="Link to this heading"></a></h3>
|
||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Create release archive</span>
|
||
<span class="w"> </span><span class="nt">run</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">|</span>
|
||
<span class="w"> </span><span class="no">tzst a release-${{ github.ref_name }}.tzst \</span>
|
||
<span class="w"> </span><span class="no">build/ docs/ \</span>
|
||
<span class="w"> </span><span class="no">--compression-level 9</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This performance guide helps you choose the right settings for your specific use case and understand how tzst compares to alternative archive tools.</p>
|
||
</section>
|
||
</section>
|
||
</section>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
||
<a href="quickstart.html" class="btn btn-neutral float-left" title="Quick Start Guide" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
|
||
<a href="examples.html" class="btn btn-neutral float-right" title="Examples" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
||
</div>
|
||
|
||
<hr/>
|
||
|
||
<div role="contentinfo">
|
||
<p>© Copyright 2026, Xi Xu.</p>
|
||
</div>
|
||
|
||
|
||
|
||
</footer>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</div>
|
||
<script>
|
||
jQuery(function () {
|
||
SphinxRtdTheme.Navigation.enable(true);
|
||
});
|
||
</script>
|
||
|
||
</body>
|
||
</html> |