网络资产信息扫描篇

乌云原帖:http://zone.wooyun.org/content/26588

网络资产信息扫描

在渗透测试(特别是内网)中经常需要对目标进行网络资产收集,即对方服务器都有哪些IP,IP上开了哪些端口,端口上运行着哪些服务,此脚本即为实现此过程,相比其他探测脚本有以下优点:1、轻巧简洁,只需python环境,无需安装额外外库。2、扫描完成后生成独立页面报告。

此脚本的大概流程为 ICMP存活探测–>端口开放探测–>端口指纹服务识别–>提取快照(若为WEB)–>生成结果报表

运行环境:python 2.7 +

README

-h 必须输入的参数,支持ip(192.168.1.1),ip段(192.168.1),ip范围指定(192.168.1.1-192.168.1.254),ip列表文件(ip.ini),最多限制一次可扫描65535个IP。
-p 指定要扫描端口列表,多个端口使用,隔开 例如:22,23,80,3306。未指定即使用内置默认端口进行扫描(21,22,23,25,53,80,110,139,143,389,443,445,465,873,993,995,1080,1723,1433,1521,3306,3389,3690,5432,5800,5900,6379,7001,8000,8001,8080,8081,8888,9200,9300,9080,9999,11211,27017)
-m 指定线程数量 默认100线程
-t 指定HTTP请求超时时间,默认为10秒,端口扫描超时为值的1/2。
-n 不进行存活探测(ICMP)直接进行扫描。

结果报告保存在当前目录(扫描IP-时间戳.html)。

例子:
python NAScan.py -h 10.111.1
python NAScan.py -h 192.168.1.1-192.168.2.111
python NAScan.py -h 10.111.1.22 -p 80,7001,8080 -m 200 -t 6
python NAScan.py -h ip.ini -p port.ini -n

服务识别在server_info.ini文件中配置
格式为:服务名|默认端口|正则 例 ftp|21|^220.*?ftp|^220-
正则为空时则使用端口进行匹配,否则以正则匹配结果为准。

代码片段

F-NAScan && server_info

F-NAScan

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
#coding:utf-8
#author:holeeinfo@gmail.com

import getopt,sys,Queue,threading,socket,struct,urllib2,time,os,re,json,base64,cgi,array,ssl

queue = Queue.Queue()
mutex = threading.Lock()
timeout = 10
port_list = []
re_data = {}
port_data = {}
statistics = {}
try:
_create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
pass
else:
ssl._create_default_https_context = _create_unverified_https_context
class UnicodeStreamFilter:
def __init__(self, target):
self.target = target
self.encoding = 'utf-8'
self.errors = 'replace'
self.encode_to = self.target.encoding
def write(self, s):
if type(s) == str:
s = s.decode("utf-8")
s = s.encode(self.encode_to, self.errors).decode(self.encode_to)
self.target.write(s)
if sys.stdout.encoding == 'cp936':
sys.stdout = UnicodeStreamFilter(sys.stdout)
class SendPingThr(threading.Thread):
def __init__(self, ipPool, icmpPacket, icmpSocket, timeout=3):
threading.Thread.__init__(self)
self.Sock = icmpSocket
self.ipPool = ipPool
self.packet = icmpPacket
self.timeout = timeout
self.Sock.settimeout(timeout + 1)

def run(self):
time.sleep(0.01)
for ip in self.ipPool:
try:
self.Sock.sendto(self.packet, (ip, 0))
except socket.timeout:
break
time.sleep(self.timeout)

class Nscan:
def __init__(self, timeout=3):
self.timeout = timeout
self.__data = struct.pack('d', time.time())
self.__id = os.getpid()

@property
def __icmpSocket(self):
Sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.getprotobyname("icmp"))
return Sock

def __inCksum(self, packet):
if len(packet) & 1:
packet = packet + '\0'
words = array.array('h', packet)
sum = 0
for word in words:
sum += (word & 0xffff)
sum = (sum >> 16) + (sum & 0xffff)
sum = sum + (sum >> 16)
return (~sum) & 0xffff

@property
def __icmpPacket(self):
header = struct.pack('bbHHh', 8, 0, 0, self.__id, 0)
packet = header + self.__data
chkSum = self.__inCksum(packet)
header = struct.pack('bbHHh', 8, 0, chkSum, self.__id, 0)
return header + self.__data

def mPing(self, ipPool):
Sock = self.__icmpSocket
Sock.settimeout(self.timeout)
packet = self.__icmpPacket
recvFroms = set()
sendThr = SendPingThr(ipPool, packet, Sock, self.timeout)
sendThr.start()
while True:
try:
ac_ip = Sock.recvfrom(1024)[1][0]
if ac_ip not in recvFroms:
log("active",ac_ip,0)
recvFroms.add(ac_ip)
except Exception:
pass
finally:
if not sendThr.isAlive():
break
return recvFroms & ipPool
def get_ac_ip(ip_list):
try:
s = Nscan()
ipPool = set(ip_list)
return s.mPing(ipPool)
except:
print 'The current user permissions unable to send icmp packets'
return ip_list
class ThreadNum(threading.Thread):
def __init__(self,queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while True:
try:
if queue.empty():break
queue_task = self.queue.get()
except:
break
try:
task_host,task_port = queue_task.split(":")
data = scan_port(task_host,task_port)
if data:
if data <> 'NULL':
port_data[task_host + ":" + task_port] = urllib2.quote(data)
server_type = server_discern(task_host,task_port,data)
if not server_type:
h_server,title = get_web_info(task_host,task_port)
if title or h_server:server_type = 'web ' + title
if server_type:log('server',task_host,task_port,server_type.strip())
except Exception,e:
continue
def get_code(header,html):
try:
m = re.search(r'<meta.*?charset\=(.*?)"(>| |\/)',html, flags=re.I)
if m:
return m.group(1).replace('"','')
except:
pass
try:
if header.has_key('Content-Type'):
Content_Type = header['Content-Type']
m = re.search(r'.*?charset\=(.*?)(;|$)',Content_Type,flags=re.I)
if m:return m.group(1)
except:
pass
def get_web_info(host,port):
h_server,h_xpb,title_str,html = '','','',''
try:
info = urllib2.urlopen("http://%s:%s"%(host,port),timeout=timeout)
html = info.read()
header = info.headers
except urllib2.HTTPError,e:
header = e.headers
except Exception,e:
return False,False
if not header:return False,False
try:
html_code = get_code(header,html).strip()
if html_code and len(html_code) < 12:
html = html.decode(html_code).encode('utf-8')
except:
pass
try:
port_data[host + ":" + str(port)] = urllib2.quote(str(header) + "\r\n\r\n" + cgi.escape(html))
title = re.search(r'<title>(.*?)</title>', html, flags=re.I|re.M)
if title:title_str=title.group(1)
except Exception,e:
pass
return str(header),title_str
def scan_port(host,port):
try:
socket.setdefaulttimeout(timeout/2)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((str(host),int(port)))
log('portscan',host,port)
except Exception,e:
return False
try:
data = sock.recv(512)
sock.close()
if len(data) > 2:
return data
else:
return 'NULL'
except Exception,e:
return 'NULL'
def log(scan_type,host,port,info=''):
mutex.acquire()
try:
time_str = time.strftime('%X', time.localtime(time.time()))
if scan_type == 'portscan':
print "[%s] %s:%d open"%(time_str,host,int(port))
try:
re_data[host].append(port)
except KeyError:
re_data[host]=[]
re_data[host].append(port)
elif scan_type == 'server':
print "[%s] %s:%d is %s"%(time_str,host,int(port),str(info))
try:
server = info.split(" ")[0].replace("(default)","")
statistics[server] += 1
except KeyError:
statistics[server] = 1
re_data[host].remove(port)
re_data[host].append(str(port) + " " + str(info))
elif scan_type == 'active':
print "[%s] %s active"%(time_str,host)
except Exception,e:
pass
mutex.release()
def read_config(config_type):
if config_type == 'server_info':
mark_list=[]
try:
config_file = open('server_info.ini','r')
for mark in config_file:
name,port,reg = mark.strip().split("|",2)
mark_list.append([name,port,reg])
config_file.close()
return mark_list
except:
print 'Configuration file read failed'
exit()
def server_discern(host,port,data):
server = ''
for mark_info in mark_list:
try:
name,default_port,reg = mark_info
if int(default_port) == int(port):server = name+"(default)"
if reg and data <> 'NULL':
matchObj = re.search(reg,data,re.I|re.M)
if matchObj:server = name
if server:
return server
except Exception,e:
continue
return server
def get_ip_list(ip):
ip_list = []
iptonum = lambda x:sum([256**j*int(i) for j,i in enumerate(x.split('.')[::-1])])
numtoip = lambda x: '.'.join([str(x/(256**i)%256) for i in range(3,-1,-1)])
if '-' in ip:
ip_range = ip.split('-')
ip_start = long(iptonum(ip_range[0]))
ip_end = long(iptonum(ip_range[1]))
ip_count = ip_end - ip_start
if ip_count >= 0 and ip_count <= 65536:
for ip_num in range(ip_start,ip_end+1):
ip_list.append(numtoip(ip_num))
else:
print '-h wrong format'
elif '.ini' in ip:
ip_config = open(ip,'r')
for ip in ip_config:
ip_list.extend(get_ip_list(ip.strip()))
ip_config.close()
else:
ip_split=ip.split('.')
net = len(ip_split)
if net == 2:
for b in range(1,255):
for c in range(1,255):
ip = "%s.%s.%d.%d"%(ip_split[0],ip_split[1],b,c)
ip_list.append(ip)
elif net == 3:
for c in range(1,255):
ip = "%s.%s.%s.%d"%(ip_split[0],ip_split[1],ip_split[2],c)
ip_list.append(ip)
elif net ==4:
ip_list.append(ip)
else:
print "-h wrong format"
return ip_list
def get_port_list(port):
if '.ini' in port:
port_config = open(port,'r')
for port in port_config:
port_list.append(port.strip())
port_config.close()
else:
port_list = port.split(',')
return port_list
def write_result():
try:
mo_html = base64.b64decode("PCFET0NUWVBFIGh0bWw+DQo8aHRtbCBsYW5nPSJlbiI+DQo8aGVhZD4NCjxtZXRhIGNoYXJzZXQ9InV0Zi04Ij4NCjxtZXRhIGh0dHAtZXF1aXY9IlgtVUEtQ29tcGF0aWJsZSIgY29udGVudD0iSUU9ZWRnZSI+DQo8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPg0KPCEtLSBUaGUgYWJvdmUgMyBtZXRhIHRhZ3MgKm11c3QqIGNvbWUgZmlyc3QgaW4gdGhlIGhlYWQ7IGFueSBvdGhlciBoZWFkIGNvbnRlbnQgbXVzdCBjb21lICphZnRlciogdGhlc2UgdGFncyAtLT4NCjx0aXRsZT7miavmj4/nu5Pmnpw8L3RpdGxlPg0KPCEtLSBCb290c3RyYXAgLS0+DQo8bGluayBocmVmPSJodHRwczovL21heGNkbi5ib290c3RyYXBjZG4uY29tL2Jvb3RzdHJhcC8zLjMuNi9jc3MvYm9vdHN0cmFwLm1pbi5jc3MiIHJlbD0ic3R5bGVzaGVldCI+DQo8IS0tIEhUTUw1IHNoaW0gYW5kIFJlc3BvbmQuanMgZm9yIElFOCBzdXBwb3J0IG9mIEhUTUw1IGVsZW1lbnRzIGFuZCBtZWRpYSBxdWVyaWVzIC0tPg0KPCEtLSBXQVJOSU5HOiBSZXNwb25kLmpzIGRvZXNuJ3Qgd29yayBpZiB5b3UgdmlldyB0aGUgcGFnZSB2aWEgZmlsZTovLyAtLT4NCjwhLS1baWYgbHQgSUUgOV0+DQogICAgICA8c2NyaXB0IHNyYz0iaHR0cHM6Ly9vc3MubWF4Y2RuLmNvbS9odG1sNXNoaXYvMy43LjIvaHRtbDVzaGl2Lm1pbi5qcyI+PC9zY3JpcHQ+DQogICAgICA8c2NyaXB0IHNyYz0iaHR0cHM6Ly9vc3MubWF4Y2RuLmNvbS9yZXNwb25kLzEuNC4yL3Jlc3BvbmQubWluLmpzIj48L3NjcmlwdD4NCiAgICA8IVtlbmRpZl0tLT4NCjxzdHlsZSB0eXBlPSJ0ZXh0L2NzcyI+DQoucG9ydCB7IGN1cnNvcjogcG9pbnRlcjsgfQ0KI2luZm9Nb2RhbCAubW9kYWwtYm9keSwgI3Jlc3BvbnNlIHsgbWF4LWhlaWdodDogNjAwcHg7IH0NCiNzdGF0Q2hhcnQgeyB3aWR0aDogMTAwJSAhaW1wb3J0YW50O2hlaWdodDogNjAwcHggIWltcG9ydGFudDsgfQ0KPC9zdHlsZT4NCjwvaGVhZD4NCjxib2R5Pg0KCTxkaXYgY2xhc3M9ImNvbnRhaW5lci1mbHVpZCI+DQoJCTxkaXYgY2xhc3M9InJvdyI+DQoJCQk8ZGl2IGNsYXNzPSJjb2wtbGctMTIiPg0KCQkJCTxkaXYgY2xhc3M9InBhbmVsIHBhbmVsLWRlZmF1bHQiPg0KCQkJCQk8ZGl2IGNsYXNzPSJwYW5lbC1oZWFkaW5nIj7mnI3liqHnu5/orqE8L2Rpdj4NCgkJCQkJPGRpdiBjbGFzcz0icGFuZWwtYm9keSI+DQoJCQkJCQk8ZGl2IGNsYXNzPSJhbGVydCBhbGVydC1zdWNjZXNzIiByb2xlPSJhbGVydCI+DQoJCQkJCQkJPHNwYW4gY2xhc3M9ImdseXBoaWNvbiBnbHlwaGljb24tdGgtbGlzdCI+PC9zcGFuPg0KCQkJCQkJCeaOoua1i+WIsOacjeWKoeWZqOaVsO+8mg0KCQkJCQkJCTxzcGFuIGlkPSJ0b3RhbC1zZXJ2ZXIiIGNsYXNzPSJiYWRnZSI+PC9zcGFuPg0KCQkJCQkJPC9kaXY+DQoJCQkJCQk8ZGl2IGNsYXNzPSJhbGVydCBhbGVydC1pbmZvIiByb2xlPSJhbGVydCI+DQoJCQkJCQkJPHNwYW4gY2xhc3M9ImdseXBoaWNvbiBnbHlwaGljb24tZmxhc2giPjwvc3Bhbj4NCgkJCQkJCQnlvIDmlL7nq6/lj6PmgLvmlbDvvJoNCgkJCQkJCQk8c3BhbiBpZD0idG90YWwtcG9ydCIgY2xhc3M9ImJhZGdlIj48L3NwYW4+DQoJCQkJCQk8L2Rpdj4NCgkJCQkJCTxkaXY+DQoJCQkJCQkJPGNhbnZhcyBpZD0ic3RhdENoYXJ0Ij48L2NhbnZhcz4NCgkJCQkJCTwvZGl2Pg0KCQkJCQk8L2Rpdj4NCgkJCQk8L2Rpdj4NCgkJCTwvZGl2Pg0KCQk8L2Rpdj4NCgkJPGRpdiBjbGFzcz0icm93Ij4NCgkJCTxkaXYgY2xhc3M9ImNvbC1sZy0xMiI+DQoJCQkJPGRpdiBjbGFzcz0icGFuZWwgcGFuZWwtZGVmYXVsdCI+DQoJCQkJCTxkaXYgY2xhc3M9InBhbmVsLWhlYWRpbmciPuaOoua1i+WIsOeahOmhtemdouagh+mimDwvZGl2Pg0KCQkJCQk8ZGl2IGNsYXNzPSJwYW5lbC1ib2R5Ij4NCgkJCQkJCTx0YWJsZSBpZD0idGl0bGUtdGFibGUiIGNsYXNzPSJ0YWJsZSB0YWJsZS1ob3ZlciI+DQoJCQkJCQkJPHRoZWFkPg0KCQkJCQkJCQk8dHI+DQoJCQkJCQkJCQk8dGg+5Zyw5Z2APC90aD4NCgkJCQkJCQkJCTx0aD7moIfpopg8L3RoPg0KCQkJCQkJCQk8L3RyPg0KCQkJCQkJCTwvdGhlYWQ+DQoJCQkJCQkJPHRib2R5Pg0KCQkJCQkJCTwvdGJvZHk+DQoJCQkJCQk8L3RhYmxlPg0KCQkJCQk8L2Rpdj4NCgkJCQk8L2Rpdj4NCgkJCTwvZGl2Pg0KCQk8L2Rpdj4NCgkJPGRpdiBjbGFzcz0icm93Ij4NCgkJCTxkaXYgY2xhc3M9ImNvbC1sZy0xMiI+DQoJCQkJPGRpdiBjbGFzcz0icGFuZWwgcGFuZWwtZGVmYXVsdCI+DQoJCQkJCTxkaXYgY2xhc3M9InBhbmVsLWhlYWRpbmciPuW8gOaUvuerr+WPozwvZGl2Pg0KCQkJCQk8ZGl2IGNsYXNzPSJwYW5lbC1ib2R5Ij4NCgkJCQkJCTx0YWJsZSBpZD0icG9ydHMtdGFibGUiIGNsYXNzPSJ0YWJsZSB0YWJsZS1ob3ZlciI+DQoJCQkJCQkJPHRoZWFkPg0KCQkJCQkJCQk8dHI+DQoJCQkJCQkJCQk8dGg+SVA8L3RoPg0KCQkJCQkJCQkJPHRoPuerr+WPozwvdGg+DQoJCQkJCQkJCTwvdHI+DQoJCQkJCQkJPC90aGVhZD4NCgkJCQkJCQk8dGJvZHk+DQoJCQkJCQkJPC90Ym9keT4NCgkJCQkJCTwvdGFibGU+DQoJCQkJCTwvZGl2Pg0KCQkJCTwvZGl2Pg0KCQkJPC9kaXY+DQoJCTwvZGl2Pg0KCTwvZGl2Pg0KCTxkaXYgY2xhc3M9Im1vZGFsIGZhZGUiIGlkPSJpbmZvTW9kYWwiIHRhYmluZGV4PSItMSI+DQoJCTxkaXYgY2xhc3M9Im1vZGFsLWRpYWxvZyBtb2RhbC1sZyI+DQoJCQk8ZGl2IGNsYXNzPSJtb2RhbC1jb250ZW50Ij4NCgkJCQk8ZGl2IGNsYXNzPSJtb2RhbC1oZWFkZXIiPg0KCQkJCQk8YnV0dG9uIHR5cGU9ImJ1dHRvbiIgY2xhc3M9ImNsb3NlIiBkYXRhLWRpc21pc3M9Im1vZGFsIj4NCgkJCQkJCTxzcGFuPiZ0aW1lczs8L3NwYW4+DQoJCQkJCTwvYnV0dG9uPg0KCQkJCQk8aDQgY2xhc3M9Im1vZGFsLXRpdGxlIj7or6bnu4bkv6Hmga88L2g0Pg0KCQkJCTwvZGl2Pg0KCQkJCTxkaXYgY2xhc3M9Im1vZGFsLWJvZHkiPg0KCQkJCQk8cHJlIGlkPSJyZXNwb25zZSI+PC9wcmU+DQoJCQkJPC9kaXY+DQoJCQkJPGRpdiBjbGFzcz0ibW9kYWwtZm9vdGVyIj4NCgkJCQkJPGJ1dHRvbiB0eXBlPSJidXR0b24iIGNsYXNzPSJidG4gYnRuLWRlZmF1bHQiIGRhdGEtZGlzbWlzcz0ibW9kYWwiPuWFs+mXrTwvYnV0dG9uPg0KCQkJCTwvZGl2Pg0KCQkJPC9kaXY+DQoJCTwvZGl2Pg0KCTwvZGl2Pg0KCTwhLS0galF1ZXJ5IChuZWNlc3NhcnkgZm9yIEJvb3RzdHJhcCdzIEphdmFTY3JpcHQgcGx1Z2lucykgLS0+DQoJPHNjcmlwdCBzcmM9Imh0dHBzOi8vY2RuanMuY2xvdWRmbGFyZS5jb20vYWpheC9saWJzL2pxdWVyeS8xLjExLjMvanF1ZXJ5Lm1pbi5qcyI+PC9zY3JpcHQ+DQoJPCEtLSBJbmNsdWRlIGFsbCBjb21waWxlZCBwbHVnaW5zIChiZWxvdyksIG9yIGluY2x1ZGUgaW5kaXZpZHVhbCBmaWxlcyBhcyBuZWVkZWQgLS0+DQoJPHNjcmlwdCBzcmM9Imh0dHBzOi8vbWF4Y2RuLmJvb3RzdHJhcGNkbi5jb20vYm9vdHN0cmFwLzMuMy42L2pzL2Jvb3RzdHJhcC5taW4uanMiPjwvc2NyaXB0Pg0KCTxzY3JpcHQgc3JjPSJodHRwczovL2NkbmpzLmNsb3VkZmxhcmUuY29tL2FqYXgvbGlicy9DaGFydC5qcy8xLjAuMi9DaGFydC5taW4uanMiPjwvc2NyaXB0Pg0KCTxzY3JpcHQgdHlwZT0idGV4dC9qYXZhc2NyaXB0Ij4NCgkJdmFyIHNlcnZlcnMgICAgPSAkc2VydmVycyQ7DQoJCXZhciBwb3J0ZGF0YSAgID0gJHBvcnRkYXRhJDsNCgkJdmFyIHN0YXRpc3RpY3MgPSAkc3RhdGlzdGljcyQ7DQoJCQ0KCQkkKGZ1bmN0aW9uKCkgew0KCQkJU3RyaW5nLnByb3RvdHlwZS5mb3JtYXQgPSBmdW5jdGlvbigpIHsNCgkJCSAgICB2YXIgcyA9IHRoaXMsaSA9IGFyZ3VtZW50cy5sZW5ndGg7DQoJCQkgICAgd2hpbGUgKGktLSkgew0KCQkJICAgICAgICBzID0gcy5yZXBsYWNlKG5ldyBSZWdFeHAoJ1xceycgKyBpICsgJ1xcfScsICdnbScpLCBhcmd1bWVudHNbaV0pOw0KCQkJICAgIH0NCgkJCSAgICByZXR1cm4gczsNCgkJCX07DQoJCQkNCgkJCXZhciBjdHggPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZCgic3RhdENoYXJ0IikuZ2V0Q29udGV4dCgiMmQiKTsNCg0KCQkJdmFyIGRhdGEgPSB7fTsNCg0KCQkJZGF0YS5sYWJlbHMgPSBPYmplY3Qua2V5cyhzdGF0aXN0aWNzKTsNCgkJCWRhdGEuZGF0YXNldHMgPSBbIHsNCgkJCQlsYWJlbCA6ICLmlbDph48iLA0KCQkJCWZpbGxDb2xvciA6ICJyZ2JhKDEwMSwxNTQsMjAxLDEpIiwNCgkJCQlzdHJva2VDb2xvciA6ICJyZ2JhKDg0LCAxNDAsIDE4OCwxKSIsDQoJCQkJaGlnaGxpZ2h0RmlsbCA6ICJyZ2JhKDEyMSwgMTg5LCAyMDYsMSkiLA0KCQkJCWhpZ2hsaWdodFN0cm9rZSA6ICJyZ2JhKDEwNSwgMTgwLCAxOTgsMSkiLA0KCQkJCWRhdGEgOiBPYmplY3Qua2V5cyhzdGF0aXN0aWNzKS5tYXAoZnVuY3Rpb24oaykgew0KCQkJCQlyZXR1cm4gc3RhdGlzdGljc1trXTsNCgkJCQl9KQ0KCQkJfSBdOw0KCQkJdmFyIHN0YXRDaGFydCA9IG5ldyBDaGFydChjdHgpLkJhcihkYXRhKTsNCgkJCSQoJyNzdGF0Q2hhcnQnKS5jbGljayhmdW5jdGlvbihldnQpIHsNCgkJCQl2YXIgYWN0aXZlQmFycyA9IHN0YXRDaGFydC5nZXRCYXJzQXRFdmVudChldnQpOw0KCQkJCSQoIiNwb3J0cy10YWJsZSB0ciIpLnJlbW92ZUNsYXNzKCk7DQoJCQkJdmFyIHR5cGUgPSBhY3RpdmVCYXJzWzBdLmxhYmVsOw0KCQkJCWlmICh0eXBlKSB7DQoJCQkJCSQoJyNwb3J0cy10YWJsZSBzcGFuW2RhdGEtb3JpZ2luYWwtdGl0bGUqPSInICsgdHlwZSArICciXScpLmNsb3Nlc3QoJ3RyJykuYWRkQ2xhc3MoJ3dhcm5pbmcnKTsNCgkJCQl9DQoJCQl9KTsNCg0KCQkJdmFyIHRvdGFsU2VydmVyID0gMCwgdG90YWxQb3J0ID0gMCwgdGl0bGVzID0gW10sIHRyZWdleCA9IC8oXGQrKSB3ZWIgKC4rKSQvOw0KCQkJJC5lYWNoKHNlcnZlcnMsIGZ1bmN0aW9uKGlwLCBwb3J0cykgew0KCQkJCXRvdGFsU2VydmVyKys7DQoJCQkJdG90YWxQb3J0ICs9IHBvcnRzLmxlbmd0aDsNCgkJCQl2YXIgb3BlbmVkID0gcG9ydHMubWFwKGZ1bmN0aW9uKHApIHsNCgkJCQkJdmFyIGFyciA9IHRyZWdleC5leGVjKHApOw0KCQkJCQlpZiAoYXJyICE9IG51bGwgJiYgYXJyLmxlbmd0aCA+IDIpIHsNCgkJCQkJCXRpdGxlcy5wdXNoKHsNCgkJCQkJCQlpcDppcCwNCgkJCQkJCQlwb3J0OmFyclsxXSwNCgkJCQkJCQl0aXRsZTphcnJbMl0NCgkJCQkJCX0pOw0KCQkJCQl9DQoJCQkJCXJldHVybiAnPHNwYW4gZGF0YS10b2dnbGU9InRvb2x0aXAiIHRpdGxlPSJ7MH0iIGNsYXNzPSJsYWJlbCBsYWJlbC1zdWNjZXNzIHBvcnQiPnsxfTwvc3Bhbj4nLmZvcm1hdChwLHAuc3BsaXQoJyAnKVswXSkNCgkJCQl9KS5qb2luKCcgJyk7DQoNCgkJCQkkKCcjcG9ydHMtdGFibGUgdGJvZHknKS5hcHBlbmQoJzx0cj48dGQ+ezB9PC90ZD48dGQ+ezF9PC90ZD48L3RyPicuZm9ybWF0KGlwLG9wZW5lZCkpOw0KCQkJfSk7DQoJCQkkKCcjdG90YWwtc2VydmVyJykuaHRtbCh0b3RhbFNlcnZlcik7DQoJCQkkKCcjdG90YWwtcG9ydCcpLmh0bWwodG90YWxQb3J0KTsNCgkJCQ0KCQkJJC5lYWNoKHRpdGxlcywgZnVuY3Rpb24oaW5kZXgsIHQpIHsNCgkJCQkkKCcjdGl0bGUtdGFibGUgdGJvZHknKS5hcHBlbmQoJzx0cj48dGQ+PGEgaHJlZj0iaHR0cDovL3swfSI+ezB9PC9hPjwvdGQ+PHRkPnsxfTwvdGQ+PC90cj4nLmZvcm1hdCh0LmlwKyc6Jyt0LnBvcnQsdC50aXRsZSkpOw0KCQkJfSk7DQoNCgkJCSQoJ1tkYXRhLXRvZ2dsZT0idG9vbHRpcCJdJykudG9vbHRpcCgpOw0KDQoJCQkkKCcucG9ydCcpLmNsaWNrKGZ1bmN0aW9uKCkgew0KCQkJCXZhciBrZXkgPSAkKHRoaXMpLnBhcmVudCgpLnByZXYoKS5odG1sKCkgKyAiOiIgKyAkKHRoaXMpLmh0bWwoKTsNCgkJCQlpZiAocG9ydGRhdGEuaGFzT3duUHJvcGVydHkoa2V5KSkgew0KCQkJCQkkKCcjcmVzcG9uc2UnKS5odG1sKGRlY29kZVVSSUNvbXBvbmVudChwb3J0ZGF0YVtrZXldKSk7DQoJCQkJCSQoJyNpbmZvTW9kYWwnKS5tb2RhbCgnc2hvdycpOw0KCQkJCX0NCgkJCX0pOw0KCQl9KTsNCgk8L3NjcmlwdD4NCjwvYm9keT4NCjwvaHRtbD4=")
mo_html = mo_html.replace('$servers$',str(json.dumps(re_data)))
mo_html = mo_html.replace('$portdata$',str(json.dumps(port_data)))
mo_html = mo_html.replace('$statistics$',str(json.dumps(statistics)))
result = open(ip + "-" + str(int(time.time())) + ".html","w")
result.write(mo_html)
result.close()
except Exception,e:
print 'Results output failure'
def t_join(m_count):
tmp_count = 0
i = 0
while True:
time.sleep(1)
ac_count = threading.activeCount()
if ac_count < m_count and ac_count == tmp_count:
i+=1
else:
i = 0
tmp_count = ac_count
#print ac_count,queue.qsize()
if (queue.empty() and threading.activeCount() <= 1) or i > 5:
break
if __name__=="__main__":
mark_list = read_config('server_info')
msg = '''
Scanning a network asset information script,author:wolf@future-sec.
Usage: python F-NAScan.py -h 192.168.1 [-p 21,80,3306] [-m 50] [-t 10] [-n]
'''

if len(sys.argv) < 2:
print msg
try:
options,args = getopt.getopt(sys.argv[1:],"h:p:m:t:n")
ip = ''
noping = False
port = '21,22,23,25,53,80,110,139,143,389,443,445,465,873,993,995,1080,1723,1433,1521,3306,3389,3690,5432,5800,5900,6379,7001,8000,8001,8080,8081,8888,9200,9300,9080,9999,11211,27017'
m_count = 100
for opt,arg in options:
if opt == '-h':
ip = arg
elif opt == '-p':
port = arg
elif opt == '-m':
m_count = int(arg)
elif opt == '-t':
timeout = int(arg)
elif opt == '-n':
noping = True
if ip:
ip_list = get_ip_list(ip)
port_list = get_port_list(port)
if not noping:ip_list=get_ac_ip(ip_list)
for ip_str in ip_list:
for port_int in port_list:
queue.put(':'.join([ip_str,port_int]))
for i in range(m_count):
t = ThreadNum(queue)
t.setDaemon(True)
t.start()
t_join(m_count)
write_result()
except Exception,e:
print e
print msg

server_info

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

ftp|21|^220.*?ftp|^220-
ssh|22|^ssh-
telnet|23|^\xff[\xfa-\xfe]|^\x54\x65\x6c
smtp|25|^220.*?smtp
dns|53|
pop3|110|\+OK.*?pop3
NetBIOS|139|
imap|143|^\* OK.*?imap
ldap|389|
smb|445|
smtps|465|
rsync|873|^@RSYNCD|^@ERROR
imaps|993|
pop3|995|\+OK
proxy|1080|
pptp|1723|
mssql|1433|
oracle|1521|
mysql|3306|^.\0\0\0.*?mysql|^.\0\0\0\n|.*?MariaDB server
rdp|3389|
svn|3690|
PostgreSql|5432|
vnc|5800|^RFB
vnc|5900|^RFB
redis|6379|-ERR|^\$\d+\r\nredis_version
Elasticsearch|9200|
Elasticsearch|9300|
memcached|11211|
mongodb|27017|
mongodb|27018|

tomcat7+nginx+memcached配置Session共享篇

一台服务器+2个tomcat7 (Nginx+tomcat7+memcached)部署负载均衡

0x01 Ngin反向代理通俗讲解+Nginx负载均衡配置(tomcat+nginx)

什么是反向代理?

反向代理(Reverse Proxy)方式是指以代理服务器来接受internet上的连接请求,然后将请求转发给内部网络上的服务器,并将从服务器上得到的结果返回给internet上请求连接的客户端,此时代理服务器对外就表现为一个服务器。

通俗的讲

这里讲得很直白。反向代理方式实际上就是一台负责转发的代理服务器,貌似充当了真正服务器的功能,但实际上并不是,代理服务器只是充当了转发的作用,并且从真正的服务器那里取得返回的数据。这样说,其实nginx完成的就是这样的工作。我们让nginx监听一个端口,譬如80端口,但实际上我们转发给在8080端口的tomcat,由它来处理真正的请求,当请求完成后,tomcat返回,但数据此时没直接返回,而是直接给nginx,由nginx进行返回,这里,我们会以为是nginx进行了处理,但实际上进行处理的是tomcat。
说到上面的方式,也许很多人又会想起来,这样可以把静态文件交由nginx来进行处理。对,很多用到nginx的地方都是作为静态伺服器,这样可以方便缓存那些静态文件,比如CSS,JS,html,htm等文件。

用到的软件nginx.exe

nginx官网下一个。http://nginx.org/en/download.html
当前我用的版本是nginx1.8.0

下完后首先要启动。进入到nginx文件夹,直接start nginx就OK了。

1
2
3
4
启动命令:start nginx
结束命令:nginx –s stop
检查命令:nignx -t
重新加载命令:nginx –s reload

基础篇详细解说请参考:http://cxshun.iteye.com/blog/1535188

打开 \conf\nginx.conf
下面是完整的配置代码

Windows上部署:一台服务器+2个tomcat 部署负载均衡Nginx+tomcat 下面是代码区域

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

#Nginx所用用户和组,window下不指定
#user nobody;
#工作的子进程数量(通常等于CPU数量或者2倍于CPU)
worker_processes 4;

#错误日志存放路径
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;

#指定pid存放文件
#pid logs/nginx.pid;

#使用网络IO模型linux建议epoll,FreeBSD建议采用kqueue,window下不指定。
events {

#允许最大连接数
worker_connections 1024;
}


http {

include mime.types;
default_type application/octet-stream;

#定义日志格式
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
#请求日志保存位置
access_log logs/access.log main;
#打开发送文件
sendfile on;
#tcp_nopush on;

#keepalive_timeout 0;
keepalive_timeout 65;
#打开gzip压缩
#gzip on;
#设定负载均衡的服务器列表
upstream local_tomcat{
server localhost:8080 weight=1;
server localhost:9090 weight=1;

}
#第一个虚拟主机
server {
#监听IP端口
listen 8081;
#主机名
server_name localhost;

#设置字符集
#charset koi8-r;
#本虚拟server的访问日志 相当于局部变量
#access_log logs/host.access.log main;

location / {
root html;
index index.html index.htm;
#此处的 http://localhost与upstream localhost对应
proxy_pass http://local_tomcat;
proxy_redirect off;
#后端的Web服务器可以通过X-Forwarded-For获取用户真实IP
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
client_max_body_size 10m; #允许客户端请求的最大单文件字节数
client_body_buffer_size 128k; #缓冲区代理缓冲用户端请求的最大字节数
proxy_connect_timeout 90; #nginx跟后端服务器连接超时时间(代理连接超时)
proxy_send_timeout 90; #后端服务器数据回传时间(代理发送超时)
proxy_read_timeout 90; #连接成功后,后端服务器响应时间(代理接收超时)
proxy_buffer_size 4k; #设置代理服务器(nginx)保存用户头信息的缓冲区大小
proxy_buffers 4 32k; #proxy_buffers缓冲区,网页平均在32k以下的话,这样设置
proxy_busy_buffers_size 64k; #高负荷下缓冲大小(proxy_buffers*2)
proxy_temp_file_write_size 64k; #设定缓存文件夹大小,大于这个值,将从upstream服务器传
}

#error_page 404 /404.html;

# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}

# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}

# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}

# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}


# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
# listen 8000;
# listen somename:8080;
# server_name somename alias another.alias;

# location / {
# root html;
# index index.html index.htm;
# }
#}


# HTTPS server
#
#server {
# listen 443 ssl;
# server_name localhost;

# ssl_certificate cert.pem;
# ssl_certificate_key cert.key;

# ssl_session_cache shared:SSL:1m;
# ssl_session_timeout 5m;

# ssl_ciphers HIGH:!aNULL:!MD5;
# ssl_prefer_server_ciphers on;

# location / {
# root html;
# index index.html index.htm;
# }
#}

}

0x02 (tomcat7+nginx+memcached) 配置Session共享

原由: 先来说一下为什么要用到memcached,因为我的一台服务器上跑2个tomcat7,用了nginx负载均衡,这个时候2个tomcat7的Session就需要共享一致。所以这个时候memcached用上了。

首页构建一个web poject项目,用eclipse新建一个web项目test,该项目下新建一个index.jsp文件,编辑该文件内容如下:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
<%@ page contentType="text/html; charset=UTF-8" %> 
<%@ page import="java.util.*" %>
<html><head><title>Cluster Test</title></head>
<body>
<%
//HttpSession session = request.getSession(true);
System.out.println(session.getId());
out.println("<br> SESSION ID:" + session.getId()+"<br>");
// 如果有新的请求,则添加session属性
String name = request.getParameter("name");
if (name != null && name.length() > 0) {
String value = request.getParameter("value");
session.setAttribute(name, value);
}
out.print("<b>Session List:</b>");
Enumeration<String> names = session.getAttributeNames();
while (names.hasMoreElements()) {
String sname = names.nextElement();
String value = session.getAttribute(sname).toString();
out.println( sname + " = " + value+"<br>");
System.out.println( sname + " = " + value);
}
%>
</body>
</html>

修改tomcat7\conf\server.xml

对应的另外两台tomcat分别是 tomcat7-1 tomcat7-2

言归正传,tomcat7+nginx+memcached 配置tomcat 集群以及负载均衡首先说一下这个memcached的jar包。我在部署的时候网上各种版本都有很气人。
jar清单:
使用的是memcached-session-manager-tc7-1.6.3 来进行的。
tomcate做session共享所需jar包
jar官网下载:http://code.google.com/p/memcached-session-manager/downloads/list

1
2
3
4
5
6
7
8
9
10
11
12
asm-3.2
javolution-5.4.3.1
kryo-1.04
kryo-serializers-0.8
memcached-session-manager-1.6.3
memcached-session-manager-tc7-1.6.3
minlog-1.2
msm-javolution-serializer-1.6.3
msm-kryo-serializer-1.6.3
msm-xstream-serializer-1.6.3
reflectasm-0.9
spymemcached-2.8.4

把上述说到的jar包下载下来后放到tomcat7/lib 目录下,修改另外2个tomcat7的配置文件tomcat7\conf\context.xml
  在标签之间加入如下代码:

1
2
3
4
5
6
7
8
<Manager
className="de.javakaffee.web.msm.MemcachedBackupSessionManager"
memcachedNodes="n1:127.0.0.1:11211"
sticky="false"
sessionBackupAsync="false"
lockingMode="uriPattern:/path1|/path2"
requestUriIgnorePattern=".*\.(ico|png|gif|jpg|css|js)$"
transcoderFactoryClass="de.javakaffee.web.msm.serializer.kryo.KryoTranscoderFactory"/>

说明:
这里的memcachedNodes是填写memcached节点,多个节点时可以以空隔分开,如: n1:localhost:11211 n2:localhost:11212 /localhost改为安装memcached的服务器的IP
sessionBackupTimeout 单位为分钟
修改后重启两个TOMCAT和nginx即可,这个时候已经解决SESSION的共享问题.

启动
然后启动memcached服务、启动nginx、启动2个tomcat7。在浏览器中输入localhost:8080/test/index.jsp 重复刷新发现session的值不变如下所示,session共享成功!
SESSION ID:76AC7C56C4B278ADFDEC6EA782192D15-n1.tomcat7-1

先写到这里,以上Windows系统上操作如上,后续补上linux的配置

0x03 centos7服务器配置 (tomcat7+nginx+memcached) 配置Session共享

上一篇是讲配置Windows篇,言归正传这一篇是讲centos7服务器上配置 Nginx+tomcat+memcached 配置Session共享

一步一步教你做

CentOS 7安装Nginx
CentOS 7安装Memcached
查看Memcached状态比较好的一款web后台管理工具是PHP写的要搭建PHP环境Memadmin

CentOS 7安装Nginx

参考借鉴:http://www.ssiyo.com/centos7_nginx1_9.html
参考借鉴:http://www.centoscn.com/nginx/2015/1007/6261.html

安装依赖的类库

1
2
3
4
5
6
7
8
9
10
11
yum install -y zlib zlib-devel openssl openssl-devel pcre pcre-deve

cd /usr/local/

mkdir nginx

cd nginx-1.9.9

./configure --prefix=/usr/local/nginx --with-http_stub_status_module

make && make install

nginx开机启动
vi /etc/rc.d/rc.local
加入如下:

1
2
/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
chmod +x /etc/rc.d/rc.local

nginx 启动

1
/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf

查看端口进程

1
2
ps -ef |grep nginx
netstat -ntlp

开始firewall端口服务

1
firewall-cmd --permanent --zone=public --add-service=http

/////////////////////////////////////////////////////////////////////
nginx 常用命令

1
2
3
/usr/local/nginx/sbin/nginx -s reload   
/usr/local/nginx/sbin/nginx -s stop
/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf

这里我安装好了以后启动nginx服务。到这个目录下面。/usr/local/nginx/sbin/
执行./nginx 就启动成功了。

0x04 Linux centos7 安装memcached

安装说明
安装环境:CentOS-7
安装方式:源码安装
软件:libevent-2.0.22-stable.tar.gz memcached-1.4.25.tar.gz
安装位置:/usr/local/libevent
/usr/local/memcached
下载地址官网
http://memcached.org/
http://libevent.org/

下载路径
wget http://www.memcached.org/files/memcached-1.4.25.tar.gz
wget https://sourceforge.net/projects/levent/files/libevent/libevent-2.0/libevent-2.0.22-stable.tar.gz

安装前提
Memcache用到了libevent这个库用于Socket的处理,所以还需要安装libevent,libevent的最新版本是libevent-2.0.22。(如果你的系统已经安装了libevent,可以不用安装)

安装libevent
将libevent-2.0.22-stable.tar.gz local创建libevent文件夹上传到/usr/local中执行以下操作:

1
2
3
4
5
6
7
8
mkdir -p /usr/local/libevent  //创建libevent目录
tar -zxvf libevent-2.0.21-stable.tar.gz // 解压压缩包
rm -rf libevent-2.0.21-stable.tar.gz // 删除压缩包
cd /libevent-2.0.21-stable

./configure -prefix=/usr/local/libevent //安装指定路径
make //编译
make install //安装

安装memcached,同时需要安装中指定libevent的安装位置:
将memcached-1.4.25.tar.gz文件上传到/usr/local中执行以下操作:

1
2
3
4
5
6
7
8
mkdir -p /usr/local/memcached //创建memcached目录
tar -zxvf memcached-1.4.25.tar.gz // 解压压缩包
rm -rf memcached-1.4.25.tar.gz // 删除压缩包
cd /usr/local/memcached-1.4.25

./configure -prefix=/usr/local/memcached --with-libevent=/usr/local/libevent 注:安装memcached时需要指定libevent的安装位置
make //编译
make install //安装

测试libevent是否安装成功:

1
[root@localhost libevent]#  ls -al /usr/local/libevent/lib/ | grep libevent

测试memcached是否安装成功:

1
[root@localhost libevent]#  ls -al /usr/local/memcached/bin/mem*

memcached必须启动才能完成服务,启动memcached:

1
[root@localhost lib]# /usr/local/memcached/bin/memcached -l 101.200.141.201 -d -p 11211 -u root -m 128  -P /usr/local/memcached/memcached.pid

具体各个参数的含义请看:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
-p 监听的端口  是设置Memcache监听的端口,这里设置11211,最好是1024以上的端口,
-l 连接的IP地址, 默认是本机 是监听的服务器IP地址,
-d start 启动memcached服务
-d restart 重起memcached服务
-d stop|shutdown 关闭正在运行的memcached服务
-d install 安装memcached服务
-d uninstall 卸载memcached服务
-u 以的身份运行 (仅在以root运行的时候有效) 是运行Memcache的用户,这里是root,
-m 最大内存使用,单位MB。默认64MB 是分配给Memcache使用的内存数量,单位是MB,这里是10MB,
-M 内存耗尽时返回错误,而不是删除项
-c 最大同时连接数,默认是1024 选项是最大运行的并发连接数,默认是1024,这里设置256,按照服务器的负载量来设定,
-f 块大小增长因子,默认是1.25-n 最小分配空间,key value flags默认是48
-h 显示帮助
-P :设置保存Memcache的pid文件 注:-P(P为大写)

结束memcached进程
如果要结束Memcache进程,执行:cat pid文件路径
kill 进程ID,具体如下

1
2
3
[root@localhost memcached]# cat memcached.pid
16741
[root@localhost memcached]# kill 16741

测试Memcached启动成功

1
2
 telnet 101.200.141.201 11211 
stats

退出当前状态:Ctrl+】+回车
telnet> quit telnet 就退出了。

防火墙开放11211端口
增加11211端口到防火墙配置中,执行以下操作:

1
[root@admin ~]# vi  /etc/sysconfig/iptables

增加以下代码

1
-A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 11211 -j ACCEPT

重启防火墙

1
[root@admin java]# service iptables restart

安装中遇到的问题
tomcat7\conf\context.xml在配置 transcoderFactoryClass=”de.javakaffee.web.msm.serializer.kryo.KryoTranscoderFactory” 如果你是从网上下载的kryo-serializers-0.8.jar版本,需要使用0.10版本,
否则报错:Caused by: java.lang.ClassNotFoundException: de.javakaffee.kryoserializers.DateSerializer
贴出下载地址:https://repo1.maven.org/maven2/de/javakaffee/kryo-serializers/0.10/

nginx+tomcat7负载均衡部署总结篇

一台服务器+2个tomcat7 (Nginx+tomcat7)部署负载均衡

什么是反向代理?

反向代理(Reverse Proxy)方式是指以代理服务器来接受internet上的连接请求,然后将请求转发给内部网络上的服务器,并将从服务器上得到的结果返回给internet上请求连接的客户端,此时代理服务器对外就表现为一个服务器。

通俗的讲

这里讲得很直白。反向代理方式实际上就是一台负责转发的代理服务器,貌似充当了真正服务器的功能,但实际上并不是,代理服务器只是充当了转发的作用,并且从真正的服务器那里取得返回的数据。这样说,其实nginx完成的就是这样的工作。我们让nginx监听一个端口,譬如80端口,但实际上我们转发给在8080端口的tomcat,由它来处理真正的请求,当请求完成后,tomcat返回,但数据此时没直接返回,而是直接给nginx,由nginx进行返回,这里,我们会以为是nginx进行了处理,但实际上进行处理的是tomcat。
说到上面的方式,也许很多人又会想起来,这样可以把静态文件交由nginx来进行处理。对,很多用到nginx的地方都是作为静态伺服器,这样可以方便缓存那些静态文件,比如CSS,JS,html,htm等文件。

用到的软件nginx.exe

nginx官网下一个。http://nginx.org/en/download.html
当前我用的版本是nginx1.8.0

下完后首先要启动。进入到nginx文件夹,直接start nginx就OK了。
启动命令:start nginx
结束命令:nginx –s stop
检查命令:nignx -t
重新加载命令:nginx –s reload

详细解说请参考:http://cxshun.iteye.com/blog/1535188

打开 \conf\nginx.conf
下面是完整的配置代码

Windows上部署:一台服务器+2个tomcat 部署负载均衡 Nginx+tomcat

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
#Nginx所用用户和组,window下不指定  
#user nobody;
#工作的子进程数量(通常等于CPU数量或者2倍于CPU)
worker_processes 4;

#错误日志存放路径
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;

#指定pid存放文件
#pid logs/nginx.pid;

#使用网络IO模型linux建议epoll,FreeBSD建议采用kqueue,window下不指定。
events {

#允许最大连接数
worker_connections 1024;
}


http {
include mime.types;
default_type application/octet-stream;

#定义日志格式
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
#请求日志保存位置
access_log logs/access.log main;
#打开发送文件
sendfile on;
#tcp_nopush on;

#keepalive_timeout 0;
keepalive_timeout 65;
#打开gzip压缩
#gzip on;
#设定负载均衡的服务器列表
upstream local_tomcat{
server localhost:8080 weight=1;
server localhost:9090 weight=1;

}
#第一个虚拟主机
server {
#监听IP端口
listen 8081;
#主机名
server_name localhost;

#设置字符集
#charset koi8-r;
#本虚拟server的访问日志 相当于局部变量
#access_log logs/host.access.log main;

location / {
root html;
index index.html index.htm;
#此处的 http://localhost与upstream localhost对应
proxy_pass http://local_tomcat;
proxy_redirect off;
#后端的Web服务器可以通过X-Forwarded-For获取用户真实IP
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
client_max_body_size 10m; #允许客户端请求的最大单文件字节数
client_body_buffer_size 128k; #缓冲区代理缓冲用户端请求的最大字节数
proxy_connect_timeout 90; #nginx跟后端服务器连接超时时间(代理连接超时)
proxy_send_timeout 90; #后端服务器数据回传时间(代理发送超时)
proxy_read_timeout 90; #连接成功后,后端服务器响应时间(代理接收超时)
proxy_buffer_size 4k; #设置代理服务器(nginx)保存用户头信息的缓冲区大小
proxy_buffers 4 32k; #proxy_buffers缓冲区,网页平均在32k以下的话,这样设置
proxy_busy_buffers_size 64k; #高负荷下缓冲大小(proxy_buffers*2)
proxy_temp_file_write_size 64k; #设定缓存文件夹大小,大于这个值,将从upstream服务器传
}

#error_page 404 /404.html;

# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}

# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}

# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}

# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}


# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
# listen 8000;
# listen somename:8080;
# server_name somename alias another.alias;

# location / {
# root html;
# index index.html index.htm;
# }
#}


# HTTPS server
#
#server {
# listen 443 ssl;
# server_name localhost;

# ssl_certificate cert.pem;
# ssl_certificate_key cert.key;

# ssl_session_cache shared:SSL:1m;
# ssl_session_timeout 5m;

# ssl_ciphers HIGH:!aNULL:!MD5;
# ssl_prefer_server_ciphers on;

# location / {
# root html;
# index index.html index.htm;
# }
#}

}

一步步教你搭dubbo+zopper+spring整合使用负载均衡容错服务

一直奔波于生活和工作中,突然发现博客近半年没用碰过了,更重要的发现是2015年只剩下几个月了……,时间真的太快太快!

先说说题外话:
这篇文章是朋友需要就帮忙捣腾了下。笔者借鉴了网上不下100篇文章才整理了如下文章,但是发现他们写的文章不是Copy别人的就是一上来就开始写过程,在这个过程中他们的文章80%一模一样的文字总感觉缺少了什么所以现在就让笔者带你进入这个过程中来..

Dubbo是什么?

Dubbo是一个分布式服务框架,致力于提供高性能和透明化的RPC远程服务调用方案,以及SOA服务治理方案,Dubbo是阿里巴巴SOA服务化治理方案的核心框架在这种情况下诞生的。

Dubbo的背景介绍?

我这就省略…了可以百度他们写的文章都Copy过我就不重复了。

Dubbo能做什么?

透明化的远程方法调用,就像调用本地方法一样调用远程方法,只需简单配置,没有任何API侵入。
软负载均衡及容错机制,可在内网替代F5等硬件负载均衡器,降低成本,减少单点。
服务自动注册与发现,不再需要写死服务提供方地址,注册中心基于接口名查询服务提供者的IP地址,并且能够平滑添加或删除服务提供者。

Ox1 第一步

什么是Zookeeper及作用?

在此笔者附上链接解释:http://www.aboutyun.com/thread-6628-1-1.html

在Windows下安装或者在Linux环境下安装zookeeper
去官网下载zookeeper-3.4.6然后安装笔者这给出官网的下载地址:http://www.apache.org/dist/zookeeper/zookeeper-3.4.6/
然后在对应的zookeeper-3.4.6/conf 下有一个文件zoo_sample.cfg的这个文件里面配置了监听客户端连接的端口等一些信息,Zookeeper 在启动时会找zoo.cfg这个文件作为默认配置文件,所以我们复制一个名称为zoo.cfg的文件,

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just
# example sakes.
dataDir=/tmp/zookeeper
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1

参数意思解释
tickTime=2000 # Zookeeper服务器心跳时间,单位毫秒
dataDir=/tmp/zookeeper # 数据持久化路径, 笔者使用的默认路径你们可以自行修改
clientPort=2181 # 连接端口
initLimit=5 # 投票选举新leader的初始化时间。
syncLimit=2 # Leader与Follower之间的最大响应时间单位,响应超过syncLimit*tickTime,Leader认为Follwer死掉,从服务器列表中删除Follwer
dataLogDir=/home/hadoop/zookeeper/logs # 日志保存路径 这个要自己新建

然后下载dubbo-admin-2.5.4.war 在Windwos或者Linux的tomcat中部署,先把dubbo-admin-2.5.4放在tomcat的webapps/ROOT下(先删除自带ROOT目录里面的内容或者备份里面的内容替换成dubbo2.5.4的内容到ROOT目录里面)。
然后进行解压:
Windows下解压:直接解压到root里面。
Linux下解压命令: #jar -xvf dubbo-admin-2.4.1.war
然后到webapps/ROOT/WEB-INF下,有一个dubbo.properties文件,里面指向Zookeeper ,使用的是Zookeeper 的注册中心,如图所示:

0x2 第二步

sping代码部分与dubbo的整合

首先做过spring项目的大家都知道先新建一个工程笔者这是spring3.1的jar包。
新建一个znn-service-provider工程

一、提供者部分
新建一个DemoService.java类,提供者和消费者都要有这个Service,

在新建一个服务提供方的实现类接口DemoServiceImpl.java,对服务消费方隐藏实现

用Spring配置声明暴露服务:
新建一个dubbo-provider.xml文件对dubbo进行配置:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:dubbo="http://code.alibabatech.com/schema/dubbo"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://code.alibabatech.com/schema/dubbo
http://code.alibabatech.com/schema/dubbo/dubbo.xsd">

<!-- 提供方应用信息,用于计算依赖关系 -->
<dubbo:application name="hello-world-app" />

<!-- 使用multicast广播注册中心暴露服务地址 -->
<!-- <dubbo:registry address="multicast://224.5.6.7:1234" /> -->

<!-- 使用zookeeper注册中心暴露服务地址 -->
<dubbo:registry address="zookeeper://182.92.228.183:2181" />

<!-- 用dubbo协议在20880端口暴露服务 -->
<dubbo:protocol name="dubbo" port="20880" />

<!-- 声明需要暴露的服务接口 -->
<dubbo:service interface="com.znn.provider.DemoService" ref="demoService" />

<!-- 和本地bean一样实现服务 -->
<bean id="demoService" class="com.znn.provider.DemoServiceImpl" />
</beans>

注:有两种暴露地址的方法,广播的那个在测试消费者的时候没有成功,就自己搭了一个zookeeper,使用zookeeper来管理。

二、消费者部分
另外新建一个工程:znn-service-consumer
导入第三方包,和提供者一样。
新建一个dubbo-consumer.xml,对dubbo进行配置:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:dubbo="http://code.alibabatech.com/schema/dubbo"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://code.alibabatech.com/schema/dubbo
http://code.alibabatech.com/schema/dubbo/dubbo.xsd">
<!-- 消费方应用名,用于计算依赖关系,不是匹配条件,不要与提供方一样 -->
<dubbo:application name="consumer-of-helloworld-app"/>
<!-- 使用multicast广播注册中心暴露发现服务地址 -->
<!-- <dubbo:registry address="multicast://224.5.6.7:1234" /> -->
<dubbo:registry address="zookeeper://182.92.228.183:2181" />
<!-- 生成远程服务代理,可以和本地bean一样使用demoService -->
<dubbo:reference id="demoService" interface="com.znn.provider.DemoService" />
</beans>

新建一个测试类实际项目中用的时候会放到特定的执行环节中去运行。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

package com.znn.consumer;

import org.springframework.context.support.ClassPathXmlApplicationContext;

import com.znn.provider.DemoService;

public class Consumer {
public static void main(String[] args) {
ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext("file:C:/Users/Dell/Desktop/dubbo/DubboDemo-master/znn-service-consumer/WebRoot/WEB-INF/dubbo-consumer.xml");
context.start();
DemoService demoService = (DemoService)context.getBean("demoService"); // 获取远程服务代理C:\Users\Dell\Desktop\dubbo\DubboDemo-master\
String hello = demoService.sayHello("world"); // 执行远程方法
System.out.println(hello);
}
}

服务器在tocmat6-7中跑起来后运行Consumer消费者测试出现以下表示服务配置正常。

然后访问dubbo_admin管理控制台页面可以看到服务器提供者上线了,用户名和密码:root,并访问服务,显示登陆页面,说明dubbo-admin2.5.4部署成功!

就先搭到这里项目中用到了的话,下一版总结过程就是在项目实际中用到遇到的问题方面的内容了!

OOAD设计原则思维导图总结篇

这是OOAD设计原则的总结思维图,有想学习OOAD思想的可以看看我的截图结合文章看更好。如果需要源码实例请在下方留言。时间就像一张网,撒在哪里,就在哪里收获。