Multiprocessing pandas apply

import multiprocessing as mp
import pandas.util.testing as pdt

def process_apply(x):
    # do some stuff to data here

def process(df):
    res = df.apply(process_apply, axis=1)
    return res

if __name__ == '__main__':
    p = mp.Pool(processes=8)
    split_dfs = np.array_split(big_df,8)
    pool_results =, split_dfs)

    # merging parts processed by different processes
    parts = pd.concat(pool_results, axis=0)

    # merging newly calculated parts to big_df
    big_df = pd.concat([big_df, parts], axis=1)

    # checking if the dfs were merged correctly
    pdt.assert_series_equal(parts['id'], big_df['id'])

PDF to TXT Python 3.5

Утащил в интернетах функцию и подправил её до рабочего состояния

ЧТобы не похерить кладу сюда)


def pdf_to_txt(filename):
from io import StringIO
from pdfminer.converter import LTChar, TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfparser import PDFDocument, PDFParser
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter

class CsvConverter(TextConverter):
def __init__(self, *args, **kwargs):
self.codec = ‘utf-8’
TextConverter.__init__(self, *args, **kwargs)

def end_page(self, i):
from collections import defaultdict
lines = defaultdict(lambda : {})
for child in self.cur_item._objs:                #<— changed
if isinstance(child, LTChar):
(_,_,x,y) = child.bbox
line = lines[int(-y)]
line[x] = child._text.encode(self.codec) #<— changed

for y in sorted(lines.keys()):
line = lines[y]
self.outfp.write(«».join(line[x].decode() for x in sorted(line.keys())))

rsrc = PDFResourceManager()
outfp = StringIO()
device = CsvConverter(rsrc, outfp,  laparams=LAParams())#codec=»utf-8″,

doc = PDFDocument()
fp = open(filename, ‘rb’)
parser = PDFParser(fp)

interpreter = PDFPageInterpreter(rsrc, device)

for i, page in enumerate(doc.get_pages()):
outfp.write(«START PAGE %d\n» % i)
if page is not None:
outfp.write(«END PAGE %d\n» % i)


return outfp.getvalue()

val = pdf_to_csv(r’С:/визуализация эволюционных каскадов сообщений в социальных сетях с помощью силовых графов.pdf’)
l = [x for x in val.split(‘\n’) if x!=’ ‘]