how_to_run.html

<!DOCTYPE html>
<html class="writer-html5" lang="en" data-content_root="./">
<head>
  <meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />

  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  <title>How to run the code &mdash; FBPIC 0.27.0 documentation</title>
      <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=03e43079" />
      <link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />

  
      <script src="_static/documentation_options.js?v=3b7790a3"></script>
      <script src="_static/doctools.js?v=9bcbadda"></script>
      <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
    <script src="_static/js/theme.js"></script>
    <link rel="index" title="Index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" />
    <link rel="next" title="Standard simulation of laser-wakefield acceleration" href="example_input/lwfa_script.html" />
    <link rel="prev" title="Installation on Summit (OLCF)" href="install/install_summit.html" /> 
</head>

<body class="wy-body-for-nav"> 
  <div class="wy-grid-for-nav">
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >

          
          
          <a href="index.html" class="icon icon-home">
            FBPIC
          </a>
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>
        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
              <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="overview/overview.html">Overview of the code</a></li>
<li class="toctree-l1"><a class="reference internal" href="install/installation.html">Installation</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">How to run the code</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#script-examples">Script examples</a><ul>
<li class="toctree-l3"><a class="reference internal" href="example_input/lwfa_script.html">Standard simulation of laser-wakefield acceleration</a></li>
<li class="toctree-l3"><a class="reference internal" href="example_input/ionization_script.html">Laser-wakefield acceleration with ionization</a></li>
<li class="toctree-l3"><a class="reference internal" href="example_input/boosted_frame_script.html">Boosted-frame simulation of laser-wakefield acceleration</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#running-the-simulation">Running the simulation</a></li>
<li class="toctree-l2"><a class="reference internal" href="#visualizing-the-simulation-results">Visualizing the simulation results</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="api_reference/api_reference.html">API reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="advanced/advanced.html">Advanced use</a></li>
</ul>

        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="index.html">FBPIC</a>
      </nav>

      <div class="wy-nav-content">
        <div class="rst-content">
          <div role="navigation" aria-label="Page navigation">
  <ul class="wy-breadcrumbs">
      <li><a href="index.html" class="icon icon-home" aria-label="Home"></a></li>
      <li class="breadcrumb-item active">How to run the code</li>
      <li class="wy-breadcrumbs-aside">
            <a href="_sources/how_to_run.rst.txt" rel="nofollow"> View page source</a>
      </li>
  </ul>
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
             
  <section id="how-to-run-the-code">
<h1>How to run the code<a class="headerlink" href="#how-to-run-the-code" title="Link to this heading">¶</a></h1>
<p>Once installed (see <a class="reference internal" href="install/installation.html"><span class="doc">Installation</span></a>), FBPIC is available as a <strong>Python
module</strong> on your system. Thus, a simulation is setup by creating a
<strong>Python script</strong> that imports and uses FBPIC’s functionalities.</p>
<section id="script-examples">
<h2>Script examples<a class="headerlink" href="#script-examples" title="Link to this heading">¶</a></h2>
<p>You can download examples of FBPIC scripts below (which you can then modify
to suit your needs):</p>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="example_input/lwfa_script.html">Standard simulation of laser-wakefield acceleration</a></li>
<li class="toctree-l1"><a class="reference internal" href="example_input/ionization_script.html">Laser-wakefield acceleration with ionization</a></li>
<li class="toctree-l1"><a class="reference internal" href="example_input/boosted_frame_script.html">Boosted-frame simulation of laser-wakefield acceleration</a></li>
</ul>
</div>
<p>(See the documentation of <a class="reference internal" href="api_reference/particles.html#fbpic.particles.fbpic.particles.Particles.Particles.make_ionizable" title="fbpic.particles.fbpic.particles.Particles.Particles.make_ionizable"><code class="xref any py py-meth docutils literal notranslate"><span class="pre">Particles.make_ionizable</span></code></a> for more information on ionization,
and the section <a class="reference internal" href="advanced/boosted_frame.html"><span class="doc">Running boosted-frame simulations</span></a> for more information on the boosted frame.)</p>
<p>The different FBPIC objects that are used in the above simulation scripts are
defined in the section <a class="reference internal" href="api_reference/api_reference.html"><span class="doc">API reference</span></a>.</p>
</section>
<section id="running-the-simulation">
<h2>Running the simulation<a class="headerlink" href="#running-the-simulation" title="Link to this heading">¶</a></h2>
<p>The simulation is then run by typing</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">python</span> <span class="n">fbpic_script</span><span class="o">.</span><span class="n">py</span>
</pre></div>
</div>
<p>where <code class="docutils literal notranslate"><span class="pre">fbpic_script.py</span></code> should be replaced by the name of your
Python script: either <code class="docutils literal notranslate"><span class="pre">lwfa_script.py</span></code> or
<code class="docutils literal notranslate"><span class="pre">boosted_frame_script.py</span></code> for the above examples.</p>
<p>If an MPI implementation is available within the compute environment and
the <code class="docutils literal notranslate"><span class="pre">mpi4py</span></code> package is installed, the computation can be scaled to multiple
processes (e.g. 4) by running</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">mpirun</span> <span class="o">-</span><span class="n">n</span> <span class="mi">4</span> <span class="n">python</span> <span class="n">fbpic_script</span><span class="o">.</span><span class="n">py</span>
</pre></div>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Note that, depending on the size of the simulation, running with
multiple MPI processes is not necessarily faster. In addition,
MPI simulations require using a finite order (e.g. <code class="docutils literal notranslate"><span class="pre">n_order=32</span></code>)
for the field solver. Please read the
documentation on the <a class="reference internal" href="overview/parallelisation.html"><span class="doc">parallelisation of FBPIC</span></a> before using this feature.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When running on CPU, <strong>multi-threading</strong> is enabled by default, and the
default number of threads is the number of (virtual) cores on your system.
You can modify this with environment variables:</p>
<ul class="simple">
<li><p>To modify the number of threads (e.g. set it to 8 threads):</p></li>
</ul>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">export</span> <span class="n">MKL_NUM_THREADS</span><span class="o">=</span><span class="mi">8</span>
<span class="n">export</span> <span class="n">NUMBA_NUM_THREADS</span><span class="o">=</span><span class="mi">8</span>
<span class="n">python</span> <span class="n">fbpic_script</span><span class="o">.</span><span class="n">py</span>
</pre></div>
</div>
<ul class="simple">
<li><p>To disable multi-threading altogether:</p></li>
</ul>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">export</span> <span class="n">FBPIC_DISABLE_THREADING</span><span class="o">=</span><span class="mi">1</span>
<span class="n">export</span> <span class="n">MKL_NUM_THREADS</span><span class="o">=</span><span class="mi">1</span>
<span class="n">export</span> <span class="n">NUMBA_NUM_THREADS</span><span class="o">=</span><span class="mi">1</span>
<span class="n">python</span> <span class="n">fbpic_script</span><span class="o">.</span><span class="n">py</span>
</pre></div>
</div>
<p>It can also happen that an alternative threading backend is selected by Numba
during installation. It is therefore sometimes required to set
<code class="docutils literal notranslate"><span class="pre">OMP_NUM_THREADS</span></code> in addition to (or instead of) <code class="docutils literal notranslate"><span class="pre">MKL_NUM_THREADS</span></code>.</p>
<p>When running in a Jupyter notebook, environment variables can be set by
executing the following command at the beginning of the notebook:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
<span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">&#39;MKL_NUM_THREADS&#39;</span><span class="p">]</span><span class="o">=</span><span class="s1">&#39;1&#39;</span>
</pre></div>
</div>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>On systems with more than one CPU socket per node, multi-threading
can become inefficient if the threads are distributed across sockets.
It can be advantageous to use one MPI process per socket and to limit the
number of threads to the number of physical cores of each socket. In addition,
it can be necessary to explicitly bind all threads of an MPI process to the
same socket.</p>
<p>On a machine with 2 sockets and 12 physical cores per socket, the following
commands spawn 2 MPI processes each with 12 threads bound to a single socket:</p>
<ul>
<li><p>Using the SLURM workload manager:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">export</span> <span class="n">MKL_NUM_THREADS</span><span class="o">=</span><span class="mi">12</span>
<span class="n">export</span> <span class="n">NUMBA_NUM_THREADS</span><span class="o">=</span><span class="mi">12</span>
<span class="n">srun</span> <span class="o">-</span><span class="n">n</span> <span class="mi">2</span> <span class="o">--</span><span class="n">cpu_bind</span><span class="o">=</span><span class="n">socket</span> <span class="n">python</span> <span class="n">fbpic_script</span><span class="o">.</span><span class="n">py</span>
</pre></div>
</div>
</li>
<li><p>Using the <code class="docutils literal notranslate"><span class="pre">mpirun</span></code> executable:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">export</span> <span class="n">MKL_NUM_THREADS</span><span class="o">=</span><span class="mi">12</span>
<span class="n">export</span> <span class="n">NUMBA_NUM_THREADS</span><span class="o">=</span><span class="mi">12</span>
<span class="n">mpirun</span> <span class="o">-</span><span class="n">n</span> <span class="mi">2</span> <span class="o">--</span><span class="n">bind</span><span class="o">-</span><span class="n">to</span> <span class="n">socket</span> <span class="n">python</span> <span class="n">fbpic_script</span><span class="o">.</span><span class="n">py</span>
</pre></div>
</div>
</li>
</ul>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When running on GPU with MPI domain decomposition, it is possible to enable
the CUDA GPUDirect technology. GPUDirect enables direct communication of
CUDA device arrays between GPUs over MPI without explicitly copying the data
to CPU first, resulting in reduced latencies and increased bandwidth. As this
feature requires a CUDA-aware MPI implementation that supports GPUDirect,
it is disabled by default and should be used with care.</p>
<p>To activate this feature, the user needs to set the following
environment variable:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">export</span> <span class="n">FBPIC_ENABLE_GPUDIRECT</span><span class="o">=</span><span class="mi">1</span>
</pre></div>
</div>
</div>
</section>
<section id="visualizing-the-simulation-results">
<h2>Visualizing the simulation results<a class="headerlink" href="#visualizing-the-simulation-results" title="Link to this heading">¶</a></h2>
<p>The code outputs HDF5 files, that comply with the
<a class="reference external" href="http://www.openpmd.org/#/start">openPMD standard</a>. As such, they
can be visualized for instance with the <a class="reference external" href="https://github.com/openPMD/openPMD-viewer">openPMD-viewer</a>). To do so, first
install the openPMD-viewer by typing</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">pip</span> <span class="n">install</span> <span class="n">openpmd</span><span class="o">-</span><span class="n">viewer</span>
</pre></div>
</div>
<p>or</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">conda</span> <span class="n">install</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="n">openpmd</span><span class="o">-</span><span class="n">viewer</span>
</pre></div>
</div>
<p>And then type</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">openPMD_notebook</span>
</pre></div>
</div>
<p>and follow the instructions in the notebook that pops up. (NB: the
notebook only shows some of the capabilities of the openPMD-viewer. To
learn more, see the tutorial notebook on the  <a class="reference external" href="https://github.com/openPMD/openPMD-viewer">Github repository</a> of openPMD-viewer).</p>
<p>If you want to render your simulation results in 3D, see the section
<a class="reference internal" href="advanced/3d_visualization.html"><span class="doc">3D visualization using PyVista</span></a>.</p>
</section>
</section>


           </div>
          </div>
          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
        <a href="install/install_summit.html" class="btn btn-neutral float-left" title="Installation on Summit (OLCF)" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
        <a href="example_input/lwfa_script.html" class="btn btn-neutral float-right" title="Standard simulation of laser-wakefield acceleration" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
    </div>

  <hr/>

  <div role="contentinfo">
    <p>&#169; Copyright 2016, FBPIC contributors.</p>
  </div>

  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
    provided by <a href="https://readthedocs.org">Read the Docs</a>.
   

</footer>
        </div>
      </div>
    </section>
  </div>
  <script>
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script> 

</body>
</html>