# Verify dir
if not os.path.isdir(dir):
    print('error: {} does not exists'.format(dir))
    sys.exit(1)

# Recursively walk thru from dir using os.walk()
for curr_dir, subdirs, files in os.walk(dir):
    # os.walk() recursively walk thru the given "dir" and its sub-directories
    # For each iteration:
    #    - curr_dir: current directory being walk thru, recursively from "dir"
    #    - subdirs: list of sub-directories in "curr_dir"
    #    - files: list of files/symlinks in "curr_dir"
    print('D:', os.path.abspath(curr_dir))    # print currently walk dir
    for subdir in sorted(subdirs):  # print all subdirs under "curr_dir"
        print('SD:', os.path.abspath(subdir))
    for file in sorted(files):      # print all files under "curr_dir"
        print(os.path.join(os.path.abspath(curr_dir), file))  # full filename

Usage: files_list [<dir>|.]
"""
import sys
import os
import glob  # Python 3.5

def main():
    # Process command-line arguments
    if len(sys.argv) > 2:  # Command-line arguments are kept in a list 'sys.argv'
        print(__doc__)
        sys.exit(1)        # Return a non-zero value to indicate abnormal termination
    elif len(sys.argv) == 2:
        dir = sys.argv[1]  # directory given in command-line argument
    else:
        dir = '.'          # default current directory

    # Check dir
    if not os.path.isdir(dir):
        print('error: {} does not exists'.format(dir))
        sys.exit(1)

    # List *.txt only
    for file in glob.glob(dir + '/**/*.txt', recursive=True):
        # ** match any files and zero or more directories and subdirectories
        print(file)

    print('----------------------------')

    # List all files and subdirs
    for file in glob.glob(dir + '/**', recursive=True):
        # ** match any files and zero or more directories and subdirectories
        if os.path.isdir(file):
            print('D:', file)
        else:
            print(file)

if __name__ == '__main__':
    main()
Copying File
shutil.copyfile(src, dest): Copy from src to dest.
Shell Command [TODO]
os.system(command_str): Run a shell command. (In Python 3, use subprocess.call() instead.)
Environment Variables [TODO]
os.getenv(varname, value=None): Returns the environment variable if it exists, or value if it doesn't, with default of None.
os.putenv(varname, value): Set environment variable to value.
os.unsetenv(varname): Delete (Unset) the environment variable.
2.3  fileinput Module
The fileinput module provides support for processing lines of input from one or more files given in the command-line arguments (sys.argv). For example, create the following script called "test_fileinput.py":

#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
"""
test_fileinput.py: Process all the files given in the command-line arguments
Usage: test_fileinput.py file1, file2, file3, ...
"""
import fileinput

def main():
    """Get lines from all the file given in the command-line arguments (sys.argv)"""
    lineNumber = 0
    for line in fileinput.input():
        line = line.rstrip()   # strip the trailing spaces/newline
        lineNumber += 1
        print('{}: {}'.format(lineNumber, line))

if __name__ == '__main__':
    main()
3.  Text Processing
For simple text string operations such as string search and replacement, you can use the built-in string functions (e.g., str.replace(old, new)). For complex pattern search and replacement, you need to master regular expression (regex).

3.1  String Operations
The built-in class str provides many member functions for text string manipulation. Suppose that s is a str object.

Strip whitespaces (blank, tab and newline)
s.strip()-> str: Return a copy of the string s with leading and trailing whitespaces removed. Whitespaces includes blank, tab and newline.
s.strip([chars]) -> str: Strip the leading/trailing characters given, instead of whitespaces.
s.rstrip(), s.lstrip() -> str: Strip the right (trailing) whitespaces and the left (leading) whitespaces, respectively.
s.rstrip() is the most commonly-used to strip the trailing spaces/newline. The leading whitespaces are usually significant.

Uppercase/Lowercase
s.upper(), s.lower() -> str: Return a copy of string s converted to uppercase and lowercase, respectively.
s.isupper(), s.islower() -> bool: Check if the string is uppercase/lowercase, respectively.
Find
s.find(key_str, [start], [end]) -> int|-1: Return the lowest index in slice s[start:end] (default to entire string); or -1 if not found.
s.index(key_str, [start], [end]) -> int|ValueError: Similar to find(), but raises ValueError if not found.
s.startswith(key_str, [start], [end]), s.endsswith(key_str, [start], [end]) -> bool: Check is the string begins or ends with key_str.
For examples,

>>> s = '/test/in.txt'
>>> s.find('in')
6
>>> s[0 : s.find('in')] + 'out.txt'
'/test/out.txt'
# You could use str.replace() described below
Find and Replace
s.replace(old, new, [count]) -> str: Return a copy with all occurrences of old replaced by new. The optional parameter count limits the number of occurrences to replace, default to all occurrences.
str.replace() is ideal for simple text string replacement, without the need for pattern matching.

For examples,

>>> s = 'hello hello hello, world'
>>> help(s.replace)
>>> s.replace('ll', '**')
'he**o he**o he**o, world'
>>> s.replace('ll', '**', 2)
'he**o he**o hello, world'
Split into Tokens and Join
s.split([sep], [maxsplit=-1]) -> [str]: Return a list of words using sep as delimiter string. The default delimiter is whitespaces (blank, tab and newline). The maxSplit limits the maximum number of split operations, with default -1 means no limit.
sep.join([str]) -> str: Reverse of split(). Join the list of string with sep as separator.
For examples,

>>> 'apple, orange, pear'.split()       # default delimiter is whitespaces
['apple,', 'orange,', 'pear']
>>> 'apple, orange, pear'.split(', ')   # Set the delimiter
['apple', 'orange', 'pear']
>>> 'apple, orange, pear'.split(', ', maxsplit=1)  # Set the split operation
['apple', 'orange, pear']

>>> ', '.join(['apple', 'orange, pear'])
'apple, orange, pear'
3.2  Regular Expression in Module re
References:

Python's Regular Expression HOWTO @ https://docs.python.org/3/howto/regex.html.
Python's re - Regular expression operations @ https://docs.python.org/3/library/re.html.
I assume that you are familiar with regex, otherwise, you could read:

"Regex By Examples" for a summary of regex syntax and examples.
"Regular Expressions" for full coverage.
The re module provides support for regular expressions (regex).

>>> import re
>>> dir(re)   # List all attributes
......
>>> help(re)  # Show man page
......
    # The man page lists all the special characters and metacharacters used by Python's regex.
Backslash (\), Python Raw String r'...' vs Regular String
Regex's syntax uses backslash (\):

for metacharacters such as \d (digit), \D (non-digit), \s (space), \S (non-space), \w (word), \W (non-word)
to escape special regex characters, e.g., \. for ., \+ for +, \* for *, \? for ?. You also need to write \\ to match \.
On the other hand, Python' regular strings also use backslash for escape sequences, e.g., \n for newline, \t for tab. Again, you need to write \\ for \.

To write the regex pattern \d+ (one or more digits) in a Python regular string, you need to write '\\d+'. This is cumbersome and error-prone.

Python's solution is using raw string with a prefix r in the form of r'...'. It ignores interpretation of the Python's string escape sequence. For example, r'\n' is '\'+'n' (two characters) instead of newline (one character). Using raw string, you can write r'\d+' for regex pattern \d+ (instead of regular string '\\d+').

Furthermore, Python denotes parenthesized back references (or capturing groups) as \1, \2, \3, ..., which can be written as raw strings r'\1', r'\2' instead of regular string '\\1' and '\\2'. Take note that some languages uses $1, $2, ... for the back references.

I suggest that you use raw strings for regex pattern strings and replacement strings.

Compiling (Creating) a Regex Pattern Object
re.compile(regexStr, [modifiers]) -> regexObj: Compile a regex pattern into a regex object, which can then be used for matching operations.
For examples,

>>> import re
>>> p1 = re.compile(r'[1-9][0-9]*|0')
      # zero or positive integers (begins with 1-9, followed by zero or more 0-9; or 0)
>>> type(p1)
<class '_sre.SRE_Pattern'>

>>> p2 = re.compile(r'^\w{6,10}$')
      # 6-10 word-character line
      # ^ matches the start-of-line, $ matches end-of-line, \w matches word character

>>> p3 = re.compile(r'xy*', re.IGNORECASE)   # with an optional modifier
      # x followed by zero or more y, case-insensitive
Invoking Regex Operations
You can invoke most of the regex functions in two ways:

regexObj.func(str): Apply compiled regex object to str, via SRE_Pattern's member function func().
re.func(regexObj|regexStr, str): Apply regex object (compiled) or regexStr (uncompiled) to str, via re's module-level function func(). These module-level functions are shortcuts to the above that do not require you to compile a regex object first, but miss the modifiers if regexStr is used.
Find using finaAll()
regexObj.findall(str) -> [str]: Return a list of all the matching substrings.
re.findall(regexObj|regexStr, str) -> [str]: same as above.
For examples,

# (1) Using compile regex object
>>> p1 = re.compile(r'[1-9][0-9]*|0')   # match integer
>>> p1.findall('123 456')
['123', '456']
>>> p1.findall('abc')
[]
>>> p1.findall('abc123xyz456_7_00')
['123', '456', '7', '0', '0']

# (2) Using re.findall() with uncompiled regex string
>>> re.findall(r'[1-9][0-9]*|0', '123 456')  # Provide the regex pattern string
['123', '456']
>>> re.findall(r'[1-9][0-9]*|0', 'abc')
[]
>>> re.findall(r'[1-9][0-9]*|0', 'abc123xyz456_7_00')
['123', '456', '7', '0', '0']
Replace using sub() and subn()
regexObj.sub(replaceStr, inStr, [count=0]) -> outStr: Substitute (Replace) the matched substrings in the given inStr with the replaceStr, up to count occurrences, with default of all.
regexObj.subn(replaceStr, inStr, [count=0]) -> (outStr, count): Similar to sub(), but return a new string together with the number of replacements in a 2-tuple.
re.sub(regexObj|regexStr, replaceStr, inStr, [count=0]) -> outStr: same as above.
re.subn(regexObj|regexStr, replaceStr, inStr, [count=0]) -> (outStr, count): same as above.
For examples,

# (1) Using compiled regex object
>>> p1 = re.compile(r'[1-9][0-9]*|0')   # match integer
>>> p1.sub(r'**', 'abc123xyz456_7_00')
'abc**xyz**_**_****'
>>> p1.subn(r'**', 'abc123xyz456_7_00')
('abc**xyz**_**_****', 5)    # (outStr, count)
>>> p1.sub(r'**', 'abc123xyz456_7_00', count=3)
'abc**xyz**_**_00'

# (2) Using re module-level function
>>> re.sub(r'[1-9][0-9]*|0', r'**', 'abc123xyz456_7_00')  # Using regexStr
'abc**xyz**_**_****'
>>> re.sub(p1, r'**', 'abc123xyz456_7_00')  # Using pattern object
'abc**xyz**_**_****'
>>> re.subn(p1, r'**', 'abc123xyz456_7_00', count=3)
('abc**xyz**_**_00', 3)
>>> re.subn(p1, r'**', 'abc123xyz456_7_00', count=10)  # count exceeds matches
('abc**xyz**_**_****', 5)
Notes: For simple string replacement, use str.replace(old, new, [max=-1]) -> str which is more efficient. See above section.

Using Parenthesized Back-References \1, \2, ... in Substitution and Pattern
In Python, regex parenthesized back-references (capturing groups) are denoted as \1, \2, .... You could use raw string (e.g., r'\1') to avoid escaping backslash in regular string (e.g., '\\1').

For examples,

# To swap the two words by using back-references in the replacement string
>>> re.sub(r'(\w+) (\w+)', r'\2 \1', 'aaa bbb ccc')
'bbb aaa ccc'
>>> re.sub(r'(\w+) (\w+)', r'\2 \1', 'aaa bbb ccc ddd')
'bbb aaa ddd ccc'
>>> re.subn(r'(\w+) (\w+)', r'\2 \1', 'aaa bbb ccc ddd eee')
('bbb aaa ddd ccc eee', 2)

# To remove duplicate words using back-reference
>>> re.subn(r'(\w+) \1', r'\1', 'hello hello world again again')
('hello world again', 2)
Find using search() and Match Object
regexObj.search(inStr, [begin], [end]) -> matchObj:
re.search(regexObj|regexStr, inStr, [begin], [end]) -> matchObj:
The search() returns a special Match object encapsulating the first match (or None if there is no matches). You can then use the following methods to process the resultant Match object:

matchObj.group(): Return the matched substring.
matchObj.start(): Return the starting matched position (inclusive).
matchObj.end(): Return the ending matched position (exclusive).
matchObj.span(): Return a tuple of (start, end) matched position.
For example,

>>> p1 = re.compile(r'[1-9][0-9]*|0')
>>> inStr = 'abc123xyz456_7_00'
>>> m = p1.search(inStr)
>>> m
<_sre.SRE_Match object; span=(3, 6), match='123'>
>>> m.group()
'123'
>>> m.span()
(3, 6)
>>> m.start()
3
>>> m.end()
6

# You can search further by providing the begin search positions
# in the form of search(str, [beginIdx]), e.g.,
>>>  m = p1.search(inStr, m.end())
>>>  m
<_sre.SRE_Match object; span=(9, 12), match='456'>

# Using a while loop
>>> m = p1.search(inStr)
>>> while m:
        print(m, m.group())
        m = p1.search(inStr, m.end())
<_sre.SRE_Match object; span=(3, 6), match='123'>
123
<_sre.SRE_Match object; span=(9, 12), match='456'>
456
<_sre.SRE_Match object; span=(13, 14), match='7'>
7
<_sre.SRE_Match object; span=(15, 16), match='0'>
0
<_sre.SRE_Match object; span=(16, 17), match='0'>
0
To retrieve the back-references (or capturing groups) inside the Match object:

matchObj.groups(): return a tuple of captured groups (or back-references)
matchObj.group(n): return the capturing group n, where n starts at 1.
matchObj.lastindex: last index of the capturing group
>>> p2 = re.compile('(A)(\w+)', re.IGNORECASE)  # Two parenthesized back-references (capturing groups)
>>> inStr = 'This is an apple.'
>>> m = p2.search(inStr)
>>> while m:
        print(m)
        print(m.group())    # show full match
        print(m.groups())   # show capturing groups in tuple
        for idx in range(1, m.lastindex + 1):  # index starts at 1
            print(m.group(idx), end=',')   # show capturing group idx
        print()
        m = p2.search(inStr, m.end())
<_sre.SRE_Match object; span=(8, 10), match='an'>
an
('a', 'n')
a,n,
<_sre.SRE_Match object; span=(11, 16), match='apple'>
apple
('a', 'pple')
a,pple,
Find using match() and fullmatch()
regexObj.match(inStr, [begin], [end]) -> matchObj:
regexObj.fullmatch(inStr, [begin], [end]) -> matchObj:
re.match(regexObj|regexStr, inStr, [begin], [end]) -> matchObj:
re.fullmatch(regexObj|regexStr, inStr, [begin], [end]) -> matchObj:
The search() matches anywhere in the given inStr[begin:end]. On the other hand, the match() matches from the start of inStr[begin:end] (similar to regex pattern ^...); while the fullmatch() matches the entire inStr[begin:end] (similar to regex pattern ^...$).

For example,

# match()
>>> p1 = re.compile(r'[1-9][0-9]*|0')   # match integers
>>> m = p1.match('aaa123zzz456')   # NOT match the beginning 'a...'
>>> m
# None
>>> m = p1.match('123zzz456')      # match the beginning (index 0)
>>> m
<_sre.SRE_Match object; span=(0, 3), match='123'>

# fullmatch()
>>> m = p1.fullmatch('123456')     # Match entire string
>>> m
<_sre.SRE_Match object; span=(0, 6), match='123456'>
>>> m = p1.fullmatch('123456abc')
>>> m
# None
Find using finditer()
regexObj.finditer(inStr) -> matchIterator
re.finditer(regexObj|regexStr, inStr) -> matchIterator
The finditer() is similar to findall(). The findall() returns a list of matched substrings. The finditer() returns an iterator to Match objects. For examples,

# Using findall()
>>> p1 = re.compile(r'[1-9][0-9]*|0')
>>> inStr = 'abc123xyz456_7_00'
>>> p1.findall(inStr)   # return a list of matched substrings
['123', '456', '7', '0', '0']
>>> for s in p1.findall(inStr):  # using for-in loop to process the list
        print(s, end=' ')
123 456 7 0 0

# Using finditer()
>>> for m in p1.finditer(inStr):  # using loop on iterator
        print(m)
<_sre.SRE_Match object; span=(3, 6), match='123'>
<_sre.SRE_Match object; span=(9, 12), match='456'>
<_sre.SRE_Match object; span=(13, 14), match='7'>
<_sre.SRE_Match object; span=(15, 16), match='0'>
<_sre.SRE_Match object; span=(16, 17), match='0'>

>>> for m in p1.finditer(inStr):
        print(m.group(), end=' ')
123 456 7 0 0
Spliting String into Tokens
regexObj.split(inStr) -> [str]:
re.split(regexObj|regexStr, inStr) -> [str]:
The split() splits the given inStr into a list, using the regex's Pattern as delimiter (separator). For example,

>>> p1 = re.compile(r'[1-9][0-9]*|0')
>>> p1.split('aaa123bbb456ccc')
['aaa', 'bbb', 'ccc']

>>> re.split(r'[1-9][0-9]*|0', 'aaa123bbb456ccc')
['aaa', 'bbb', 'ccc']
Notes: For simple delimiter, use str.split([sep]), which is more efficient. See above section.

4.  Web Scraping
References:
Beautiful Soup Documentation @ https://www.crummy.com/software/BeautifulSoup/bs4/doc/.
Web Scraping (or web harvesting or web data extraction) refers to reading the raw HTML page to retrieve desired data. Needless to say, you need to master HTML, CSS and JavaScript.

Python supports web scraping via packages requests and BeautifulSoup (bs4).

Install Packages
You could install the relevant packages using pip as follows:

$ pip install requests
$ pip install bs4
Step 0: Inspect the Target Webpage
Press F12 on the target webpage to turn on the "F12 debugger".
Choose "Inspector".
Click the "Select" (the left-most icon with a arrow) and point your mouse at the desired part of the HTML page. Study the codes.
Step 1: Send a HTTP GET request to the target URL to retrieve the raw HTML page using module requests
>>> import requests

# Set the target URL
>>> url = "http://your_target_webpage"

# Send a HTTP GET request to the target URL to retrieve the HTML page,
# which returns a "Response" object.
>>> response = requests.get(url)

# Inspect the "Response' object
>>> type(response)
<class 'requests.models.Response'>
>>> response
<Response [200]>
>>> help(response)
......
>>> print(response.text)     # content of the response, in unicode text
......
>>> print(response.content)  # content of the response, in raw bytes
......
Step 2: Parse the HTML Text into a Tree-Structure using BeautifulSoup and Search the Desired Data
# Continue from Step 1
>>> from bs4 import BeautifulSoup

# Parse the HTML text into a tree-structure "BeautifulSoup" object
>>> soup = BeautifulSoup(response.text, "html.parser")

# Inspect the "BeautifulSoup" object
>>> type(soup)
<class 'bs4.BeautifulSoup'>
>>> help(soup)
......

# Find the first appearance a particular HTML tag (e.g, 'img') via find(tag) -> str
>>> img_tag = soup.find('img')
>>> img_tag
<img ...... />

# Find all the appearances of a particular tag, via findAll(tag) -> [str]
>>> img_tags = soup.findAll('img')
>>> img_tags
[<img ... />, <img ... />, <img ... />, ...]

# Find the first appearance of a tag with certain attributes
>>> soup.find('div', attrs = {'id':'test'})
# Find all appearances of a tag with certain attributes
>>> soup.findAll('div', attrs = {'class':'error'})
You could write out the selected data to a file:

with open(filename, 'w') as fp:
    for row in rows:
        fp.wrire(row + '\n')
You could also use csv module to write out rows of data with a header:

>>> import csv
>>> with open(filename, 'w') as fp:
        writer = csv.DictWriter(fp, ['colHeader1', 'colHeader2', 'colHeader3'])
        writer.writeheader()
        for row in rows:
            writer.writerow(row)
Step 3: Download Selected Document Using urllib.request
You may want to download documents such as text files or images.

# Continue from Step 2
>>> import urllib.request
# Download a select URL and save the response in a local file
>>> downloadUrl = '.....'
>>> file = '......'
>>> urllib.request.urlretrieve(download_url, file)
Step 4: Delay
To avoid spamming a website with download requests (and flagged as a spammer), you need to pause your code for a while.

>>> import time
# Pause (in seconds)
>>> time.sleep(1)
REFERENCES & RESOURCES