Code Template
Python多进程写入文件(multiprocessing template)
1#multiprocessing template
2import multiprocessing
3
4def write_file(result):
5 f = open('')
6 f.write(result)
7
8def process(args):
9 '''
10 work
11 '''
12 return result
13
14if __name__ == '__main__':
15 pool = multiprocessing.Pool(processes = 100)
16 for t in tasks:
17 pool.apply_async(process, (t, ),callback=write_file)
18 pool.close()
19 pool.join()
高次方取模
次方求模模板
就是求(a^b)% p
其中 a, b, p 数值比较大
取余公式:(a*b)% p = (a%p * b%p)% p
1//次方求模模板
2template<class Type>
3inline Type ModPow(Type m,Type n,Type p) //m的n次方模p
4{
5 if(n==0) return 1;
6 if (n==1) return m%p;
7 Type tmp=ModPow(m,n>>1,p);
8 return (tmp*tmp%p)*((n%2?m:1)%p)%p;
9}
高次方求模
1/*高次方求模:
2比如a的b次方对c求模
3我们可以把b 化为二进制形式看那一位有1
4比如b=10101则 a^b=a^(10000)*a^(100)*a^(1)
5以函数形式体现:*/
6template<class Type>
7inline Type han()
8{
9 Type t,s;
10 for(t=a,s=1;b;b>>=1,t*=t,t%=c)//用b>>=1查看b中1
11 if(b&1){s*=t;s%=c;}
12 return s%c;
13}
大数取模的模板
取余公式:(a+b)% p = (a%p+b%p)% p;
1char big_number[MAX];
2template<class type>
3type big_mod(char *a,type mod){
4 type len=strlen(a),result=0;
5 for(int i=0;i<len;i++){
6 result=((result*10)%mod+(a[i]-'0')%mod)%mod;
7 //如果数据不算太大,可以写成这样来节省时间result=(result*10+a[i]-'0')%mod;
8 }
9 return result;
10}
11//test
12char s[200];
13int main()
14{ while(gets(s))
15 cout<<big_mod<int>(s,2)<<endl;
16}
哈希模版(hash template)
1const int MAX=1000003;
2template <class T>
3class hash
4{
5private:
6 int pos;
7 int next[MAX];
8 int head[MAX];
9 T key[MAX];
10public:
11 hash();
12 bool search(T x);
13 void push(T x);
14};
15template <class T>
16hash<T>::hash()
17{
18 pos=0;
19 memset(next,-1,sizeof(next));
20 memset(head,-1,sizeof(head));
21 //memset(key,-1,sizeof(key));
22}
23template <class T>
24inline bool hash<T>::search(const T x)
25{
26 int temp=x%MAX;
27 int t=head[temp];
28 while(t!=-1)
29 {
30 if (key[t]==x)
31 {
32 return 1;
33 }
34 t=next[t];
35 }
36 return 0;
37}
38template <class T>
39inline void hash<T>::push(const T x)
40{
41 int temp=x% MAX;
42 if (head[temp]!=-1)
43 {
44 next[pos]=head[temp];
45 }
46 head[temp]=pos;
47 key[pos]=x;
48 pos++;
49}
Python 协程处理 requests
1from gevent import monkey
2monkey.patch_all()
3
4import sys
5reload(sys)
6sys.setdefaultencoding('utf-8')
7
8import requests
9import time
10import gevent
11from gevent import pool
12import re
13import math
14import pymysql
15import functools
16from DBUtils.PooledDB import PooledDB
17from datetime import datetime
18from collections import defaultdict
19
20# db ip
21'''
22db msg
23'''
24# 数据库连接池
25mysql_db_pool = PooledDB(creator=pymysql, mincached=1, maxcached=2, maxconnections=100, host=base_ip, port=3306,
26 user=base_user, passwd=base_pwd, db=base_db, charset='utf8', use_unicode=False, blocking=True)
27# target url
28URL = ''
29
30
31complete_count = 0
32# log--
33func_count_dict = defaultdict(int)
34
35def func_time_logger(fun):
36 if fun.__dict__.get('mioji.aop_utils.logger', False):
37 return fun
38 fun.__dict__['mioji.aop_utils.logger'] = True
39
40 @functools.wraps(fun)
41 def logging(*args, **kw):
42 func_count_dict[fun.__name__] += 1
43 begin = datetime.now()
44 result = fun(*args, **kw)
45 end = datetime.now()
46 func_count_dict[fun.__name__] -= 1
47 print 'func {0}, cost:{1}, ex:{2}'.format(fun.__name__, end - begin , func_count_dict[fun.__name__])
48 return result
49
50 return logging
51# --log
52
53# 获得代理
54@func_time_logger
55def get_PROXY():
56 # 代理url
57 R = requests.get(url)
58 p = R.content
59 if p.startswith('10.'):
60 # if p.split(':')[0] in SOCKS_PROXY:
61 PROXY = {
62 'http': 'socks5://' + p,
63 'https': 'socks5://' + p
64 }
65 else:
66 self.real_ip = p.split(':')[0]
67 proxy_type = 'http'
68 PROXY = {
69 'https': 'http://' + p,
70 'http': 'http://' + p,
71 }
72
73 return PROXY
74
75# requests
76@func_time_logger
77def get_request_data(x, y):
78 i = 0
79 while i < 3:
80 try:
81 r = requests.get(URL, proxies=get_PROXY(), timeout=(5, 10))
82 data = r.content
83 return data
84 except:
85 print 'get error'
86 i += 1
87 return None
88
89# process function
90@func_time_logger
91def process(a1,a2):
92
93 get_request_data(x,y)
94
95 '''
96 other works
97 '''
98
99 global complete_count
100 complete_count += 1
101 print 'end {0}'.format(sid)
102 print 'complete {0}'.format(complete_count)
103
104# 异步协程处理请求量大的需求
105if __name__ == '__main__':
106 s = time.time()
107
108 gs = []
109 for data in data_list:
110 g = execute_pool.apply_async(process, args=(a1, a2,))
111 gs.append(g)
112 gevent.joinall(gs)
113
114 print time.time() - s
115