pipsi

apt-get install python3-venv
sudo pip3 install virtualenv wheel
curl https://raw.githubusercontent.com/mitsuhiko/pipsi/master/get-pipsi.py | python3

pipsi install --python /usr/bin/python3.7 hovercraft
pipsi install --python /usr/bin/python3.7 -e .

Bootstrap project

cookiecutter gh:mwicat/cookiecutter-pypackage

Bootstrap project from config

cookiecutter gh:mwicat/cookiecutter-pypackage --config-file cookiecutter-mypackage.yaml --overwrite-if-exists --no-input

create virtualenv

python3 -m venv yourvenv

source yourvenv/bin/activate

install apps from virtualenvs

python3 -m pip install --user pipx
python3 -m userpath append ~/.local/bin

install apps

pipx install PACKAGE

list apps

pipx list

export PATH="$PATH:$HOME/.local/bin/mgr"

chain exceptions

from six import raise_from
raise_from(new_exception, original_exception)

uwsgi app

def application(env, start_response):
    start_response('200 OK', [('Content-Type','text/html')])
    return [b"Hello World"]

uwsgi --http :9090 --wsgi-file foobar.py

Progressbar

sudo pip install progressbar2

import progressbar

WIDGETS = [
    progressbar.FormatLabel('%(value)d/%(max)d'), ' ',
    progressbar.Percentage(), ' ',
    progressbar.Bar(), ' ',
    progressbar.FileTransferSpeed(unit='lines'), ' ',
    progressbar.ETA()
]


def pbar(seq, *args, **kw):
    kw['widgets'] = WIDGETS
    return progressbar.ProgressBar(*args, **kw)(seq)

lines = 944567

for row in pbar(rows, max_value=lines):
    continue

import progressbar

with progressbar.ProgressBar(max_value=10) as bar:
    for i in range(10):
        time.sleep(0.1)
        bar.update(i)

Remote debugging

pip install remote-pdb

from remote_pdb import set_trace; set_trace(port=4444)

or:

export REMOTE_PDB_PORT=4444

from remote_pdb import set_trace; set_trace()

Reload module list

import pkg_resouces; pkg_resources.get_distribution('winpdb2').activate()

Python 2.5 Compilation

http://www.talino.org/tutorials/install-python-261-without-trashing-ubuntu/

Edit ./Modules/Setup
Uncomment zlib line

sudo aptitude install build-essential libncursesw5-dev libreadline5-dev libssl-dev libgdbm-dev libbz2-dev libc6-dev libsqlite3-dev libdb-dev tk-dev

Fabric need Paramiko:

on windows - http://www.voidspace.org.uk/python/modules.shtml#pycrypto

Arguments parsing

from argh import *

@arg('text', default='hello world')
def another_echo(text):
    print text

parser = ArghParser()
parser.add_commands([echo, another_echo])

if __name__ == '__main__':
    parser.dispatch()

IPython

from IPython import embed; embed()

CPU Profiling

python -m cProfile -o profile_data.pyprof sql_distribution.py
sudo pip install pyprof2calltree
sudo apt-get install kcachegrind
pyprof2calltree -i profile_data.pyprof -k

Tornado timeout

tornado.ioloop.IOLoop.instance().add_timeout(time.time() + 2, handler)

Memory profiling

• Heapy tutorial
• Hunting memleaks
• Dowser

from guppy import hpy
hp=hpy()
h=hp.heap()
h.byrcs

Torando profiling

from guppy import hpy; hp=hpy()

def stop_handler():
    print 'stopping tornado'
    IOLoop.instance().stop()
    print 'snapshot'
    h = hp.heap()
    print 'starting ipython'
    from IPython import embed; embed()

Fabric

import paramiko
paramiko.util.log_to_file('ssh.log')

Print object count on signal

import signal, gc

def debug(sig, frame):
    gc.collect()
    count0, count1, count2 = gc.get_count()
    count = count0 + count1 + count2
    statsd.gauge('memory.object_count', count)

signal.signal(signal.SIGUSR1, debug)  # Register handler

Print stack trace on signal

import code, traceback, signal, sys

def debug(sig, frame):
    """Interrupt running process, and provide a python prompt for
    interactive debugging."""
    d={'_frame':frame}         # Allow access to frame object.
    d.update(frame.f_globals)  # Unless shadowed by global
    d.update(frame.f_locals)

    message  = "Signal recieved : entering python shell.\nTraceback:\n"
    message += ''.join(traceback.format_stack(frame))
    sys.exit(1)

signal.signal(signal.SIGUSR1, debug)  # Register handler

Too small /tmp

mkdir -p $HOME/tmp
export TMPDIR=$HOME/tmp
export PIP_DOWNLOAD_CACHE=$HOME/tmp

Remote console

sudo pip install rfoo

S:

from rfoo.utils import rconsole
rconsole.spawn_server()

C: rconsole

Debugging

import pdb; pdb.set_trace()

Remote debugging

port: 51000 http://winpdb.org/docs/embedded-debugging/

sudo pip install winpdb

S: rpdb2 -s -d -r script.py

or

import rpdb2; rpdb2.start_embedded_debugger('haslo', fAllowRemote=True, timeout=60)

C: winpdb -olocalhost -a script.py

Get all instances of a class

instances = [obj for obj in gc.get_objects() if isinstance(obj, data.bcspell.BCspell)]

gc.get_referrers(instances[0])

Reuse Twisted protocols

from twisted.protocols import basic
from twisted.internet import protocol
rec = basic.NetstringReceiver()

from StringIO import StringIO
s = StringIO()
transport = protocol.FileWrapper(s)
rec.makeConnection(transport)
rec.sendString('marek')
s.getvalue()

Pyro

With nameserver

python -m Pyro4.naming

import Pyro4, nltk
Pyro4.Daemon.serveSimple({nltk.tokenize: 'tokenize'}, ns=True)

import Pyro4
tokenize=Pyro4.Proxy('PYRONAME:tokenize')
tokenize.word_tokenize('fajnie, co nie?')

Without nameserver

import Pyro4, nltk
Pyro4.Daemon.serveSimple({nltk.tokenize: 'tokenize'}, ns=False, port=4433)

import Pyro4
tokenize=Pyro4.Proxy('PYRO:tokenize@localhost:4433')
tokenize.word_tokenize('fajnie, co nie?')

Current traceback

import traceback; print ''.join(traceback.format_stack())

Time

import time; time.strftime("%d%m%y_%H%M%S", time.localtime())

Parse dates

from datetime import datetime
s = '2018-04-09 15:00:00'
dt = datetime.strptime(s, '%Y-%m-%d %H:%M:%S')

sudo pip install python-dateutil

from dateutil.parser import parse as parse_date
parse_date('21 Aug')

sudo pip install python-datemath

from datemath import parse as parse_datemath
parse_datemath('now-2d')

from dateutil.parser import parse as parse_date
from datemath import parse as parse_datemath

try:
    start_dt = parse_date(start)
except ValueError:
    start_dt = parse_datemath(start)

start_ts = time.mktime(start_dt.timetuple())

Format date

ISO8601

from datetime import datetime
import pytz

ts = 1463288494
tz = pytz.timezone('GMT')
print(datetime.fromtimestamp(ts, tz).isoformat())

import time
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(1347517370))

setup.py

#!/usr/bin/env python

from setuptools import find_packages, setup

setup(name='relmaker',
      version='0.0.1',
      description='a tool',
      author='mwicat',
      author_email='mwicat at gmail.com',
      url='http://URL',
      packages=find_packages(),
      package_data={'relmaker': ['templates/*']},
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          'argh>=0.24.1',
          'argcomplete',
          'requests',
          'pygments',
      ],
      entry_points={'console_scripts': [
          'relmaker = relmaker.main:main'],
      })

syntax color pygments

from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import get_formatter_by_name

def hl(data, format):
    return highlight(data, get_lexer_by_name(format), get_formatter_by_name('terminal'))

Parse url query

import urllib
url = 'http://foo.appspot.com/abc?def=ghi'
parsed = urllib.parse.urlparse(url)
u = urllib.parse.parse_qs(parsed.query)

from furl import furl
url = 'http://www.google.com/?one=1&two=2'
f = furl(url)
print f.args['three']

Encode url query

urllib.parse.urlencode(f)

Parse html

http://lxml.de/3.7/api/index.html

sudo pip install lxml cssselect

from lxml import html
url = 'http://econpy.pythonanywhere.com/ex/001.html'
params = {'a': 2}
page = requests.get(url, params=params)
tree = html.fromstring(page.content)
for div in tree.cssselect('a'):
    print '%s: %s' % (div.text_content(), div.get('href'))

Configuration file

import ConfigParser
import os

RC_NAME = '.nagiosquery'
RC_FILE = os.path.join(os.path.expanduser('~'), RC_NAME)

cfg = ConfigParser.ConfigParser()

if os.path.exists(RC_FILE):
    cfg.read(RC_FILE)

try:
    username = cfg.get('credentials', 'username')
    password = cfg.get('credentials', 'password')
except ConfigParser.Error:
    username = None
    password = None

Module directory path

BASE_DIR = os.path.dirname(os.path.abspath(__file__))

Debug urllib/requests

try:
    import http.client as http_client
except ImportError:
    import httplib as http_client
http_client.HTTPConnection.debuglevel = 1

Decompile

sudo pip install uncompyle6

uncompyle6 -o src -r dir1 dir2

Timed seq

def timed_seq(seq, rate):
    last_sec = int(time.time())
    curr_cnt = 0
    it = iter(seq)

    while True:
        curr_sec = int(time.time())
        if curr_sec != last_sec:
            curr_cnt = 0
            last_sec = curr_sec
        if curr_cnt < rate:
            yield next(it)
            curr_cnt += 1
        time.sleep(0.001)

jinja templates

sudo pip install Jinja2

from jinja2 import Template

tpl = open('template.jinja2').read()
template = Template(tpl, trim_blocks=True, lstrip_blocks=True)

domains = [l.strip().split() for l in open('domains.lst')]
domains = [{'name' : d[0], 'sub': len(d) > 1} for d in domains]

print(template.render(domains=domains))

Parse number with commas

import locale
locale.setlocale(locale.LC_ALL, '')
locale.atof('100,050.23')

py2to3

2to3 --output-dir=mycode-py3 -W -n mycode

  -W, --write-unchanged-files
                        Also write files even if no changes were required
                        (useful with --output-dir); implies -w.
  -n, --nobackups       Don't write backups for modified files