banners (1ac9989a) · Commits · nimrod / masscan

Makefile

+1 −1

Original line number	Diff line number	Diff line
		@@ -7,7 +7,7 @@ SYS := $(shell gcc -dumpmachine)
		# environment where things likely will work -- as well as anything
		# works on the bajillion of different Linux environments
		ifneq (, $(findstring linux, $(SYS)))
		LIBS = -lpcap -lm -lrt -ldl -rdynamic
		LIBS = -lpcap -lm -lrt -ldl -rdynamic -lpthread
		INCLUDES = -I. -I../PF_RING/userland/lib
		FLAGS2 =
		endif

src/main-conf.c

+10 −1

Original line number	Diff line number	Diff line
		@@ -15,6 +15,7 @@
		#include "ranges.h"
		#include "string_s.h"
		#include "logger.h"
		#include "proto-banner1.h"

		#include <ctype.h>
		#include <limits.h>
		@@ -606,6 +607,11 @@ masscan_set_parameter(struct Masscan masscan, const char name, const char *val
		masscan->nmap.append = 1;
		} else if (EQUALS("badsum", name)) {
		masscan->nmap.badsum = 1;
		} else if (EQUALS("banner1", name)) {
		banner1_test(value);
		exit(1);
		} else if (EQUALS("banners", name)) {
		masscan->is_banners = 1;
		} else if (EQUALS("datadir", name)) {
		strcpy_s(masscan->nmap.datadir, sizeof(masscan->nmap.datadir), value);
		} else if (EQUALS("data-length", name)) {
		@@ -824,6 +830,7 @@ is_singleton(const char *name)
		"no-stylesheet",
		"send-eth", "send-ip", "iflist", "randomize-hosts",
		"nmap", "trace-packet", "pfring", "sendq",
		"banners",
		0};
		size_t i;

		@@ -951,7 +958,9 @@ masscan_command_line(struct Masscan masscan, int argc, char argv[])
		masscan_set_parameter(masscan, "includefile", argv[++i]);
		break;
		case 'R':
		fprintf(stderr, "nmap(%s): quasi-supported, see documentation\n", argv[i]);
		/* -iR in nmap makes it randomize addresses completely. Thus,
		* it's nearest equivelent is scanning the entire Internet range */
		masscan_set_parameter(masscan, "include", "0.0.0.0/0");
		break;
		default:
		fprintf(stderr, "nmap(%s): unsupported option\n", argv[i]);

src/main.c

+198 −48

Original line number	Diff line number	Diff line
		@@ -18,11 +18,13 @@
		#include "main-throttle.h" /* rate limit */
		#include "main-dedup.h" /* ignore duplicate responses */
		#include "proto-arp.h" /* for responding to ARP requests */
		#include "proto-banner1.h"
		#include "proto-tcp.h" /* for TCP/IP connection table */
		#include "syn-cookie.h" /* for SYN-cookies on send */
		#include "output.h" /* for outputing results */
		//#include "xring.h" /* producer/consumer ring buffer */
		#include "rte-ring.h" /* producer/consumer ring buffer */
		#include "rawsock-pcapfile.h" /* for saving pcap files w/ raw packets */
		#include "smack.h" /* Aho-corasick state-machine pattern-matcher */

		#include "pixie-timer.h" /* portable time functions */
		#include "pixie-threads.h" /* portable threads */
		@@ -39,27 +41,67 @@ unsigned control_c_pressed = 0;
		time_t global_now;



		/***************************************************************************
		* The recieve thread doesn't transmit packets. Instead, it queues them
		* up on the transmit thread. Every so often, the transmit thread needs
		* to flush this transmit queue and send everything.
		*
		* This is an inherent design issue trying to send things as batches rather
		* than individually. It increases latency, but increases performance. We
		* don't really care about latency.
		***************************************************************************/
		void
		flush_packets(struct Masscan *masscan)
		flush_packets(struct Masscan masscan, struct Throttler throttler, uint64_t *packets_sent)
		{
		for (;;) {
		unsigned char *p;
		uint64_t batch_size;

		/*
		* Only send a few packets at a time, throttled according to the max
		* --rate set by the usser
		*/
		batch_size = throttler_next_batch(throttler, *packets_sent);

		/*
		* Send a batch of queued packets
		*/
		for ( ; batch_size; batch_size--) {
		int err;
		struct PacketBuffer *p;

		err = rte_ring_sc_dequeue(masscan->pending_packets, (void**)&p);
		/*
		* Get the next packet from the transmit queue. This packet was
		* put there by a receive thread, and will contain things like
		* an ACK or an HTTP request
		*/
		err = rte_ring_sc_dequeue(masscan->transmit_queue, (void**)&p);
		if (err)
		break;
		rawsock_send_packet(masscan->adapter, p + sizeof(size_t), (unsigned)(size_t)p, 0);
		break; /* queue is empty, nothing to send */

		/*
		* Actually send the packet
		*/
		rawsock_send_packet(masscan->adapter, p->px, (unsigned)p->length, 1);

		/*
		* Now that we are done with the packet, put it on the free list
		* of buffers that the transmit thread can reuse
		*/
		for (err=1; err; ) {
		err = rte_ring_sp_enqueue(masscan->packet_buffers, p);
		again2:
		if (err) {
		LOG(0, "transmit queue full (should be impossible)\n");
		pixie_usleep(10000);
		goto again2;
		}
		}


		/*
		* Remember that we sent a packet, which will be used in
		* throttling.
		*/
		(*packets_sent)++;
		}
		}


		@@ -87,6 +129,7 @@ transmit_thread(void v) /aka. scanning_thread() */
		double timestamp_start;
		unsigned *picker;
		struct Adapter *adapter = masscan->adapter;
		uint64_t packets_sent = 0;

		LOG(1, "xmit: starting transmit thread...\n");

		@@ -123,7 +166,8 @@ transmit_thread(void v) /aka. scanning_thread() */
		* per-packet cost by doing batches. At slower rates, the batch
		* size will always be one.
		*/
		batch_size = throttler_next_batch(&throttler, i);
		batch_size = throttler_next_batch(&throttler, packets_sent);
		packets_sent += batch_size;
		while (batch_size && i < m) {
		unsigned ip;
		unsigned port;
		@@ -166,7 +210,7 @@ transmit_thread(void v) /aka. scanning_thread() */
		} /* end of batch */

		/* Transmit packets from other thread */
		flush_packets(masscan);
		flush_packets(masscan, &throttler, &packets_sent);

		/* If the user pressed <ctrl-c>, then we need to exit. but, in case
		* the user wants to resume the scan later, we save the current
		@@ -196,7 +240,7 @@ transmit_thread(void v) /aka. scanning_thread() */

		for (k=0; k<1000; k++) {
		/* Transmit packets from other thread */
		flush_packets(masscan);
		flush_packets(masscan, &throttler, &packets_sent);

		pixie_usleep(1000);
		}
		@@ -211,6 +255,13 @@ transmit_thread(void v) /aka. scanning_thread() */


		/***************************************************************************
		*
		* Asynchronous receive thread
		*
		* The transmit and receive threads run independently of each other. There
		* is no record what was transmitted. Instead, the transmit thread sets a
		* "SYN-cookie" in transmitted packets, which the receive thread will then
		* use to match up requests with responses.
		***************************************************************************/
		static void
		receive_thread(struct Masscan *masscan,
		@@ -221,6 +272,9 @@ receive_thread(struct Masscan *masscan,
		struct Output *out;
		struct DedupTable *dedup;
		struct PcapFile *pcapfile = NULL;
		struct TCP_ConnectionTable *tcpcon = 0;



		/*
		* If configured, open a pcap file for saving raw packets. This is
		@@ -228,7 +282,7 @@ receive_thread(struct Masscan *masscan,
		* strange things people send us. Note that we don't record transmitted
		* packets, just the packets we've received.
		*/
		if (masscan->pcap_filename)
		if (masscan->pcap_filename[0])
		pcapfile = pcapfile_openwrite(masscan->pcap_filename, 1);

		/*
		@@ -242,6 +296,20 @@ receive_thread(struct Masscan *masscan,
		*/
		dedup = dedup_create();

		/*
		* Create a TCP connection table for interacting with live
		* connections
		*/
		if (masscan->is_banners) {
		tcpcon = tcpcon_create_table(
		(size_t)(masscan->max_rate/5),
		masscan->transmit_queue,
		masscan->packet_buffers,
		masscan->pkt_template,
		output_report_banner,
		out
		);
		}

		/*
		* Receive packets. This is where we catch any responses and print
		@@ -257,9 +325,10 @@ receive_thread(struct Masscan *masscan,
		int err;
		unsigned x;
		struct PreprocessedInfo parsed;
		unsigned dst;
		unsigned src;
		unsigned seqno;
		unsigned ip_me;
		unsigned ip_them;
		unsigned seqno_them;
		unsigned seqno_me;

		err = rawsock_recv_packet(
		masscan->adapter,
		@@ -271,24 +340,33 @@ receive_thread(struct Masscan *masscan,
		if (err != 0)
		continue;

		/*
		* Do any TCP event timeouts based on the current timestamp from
		* the packet. For example, if the connection has been open for
		* around 10 seconds, we'll close the connection.
		*/
		if (tcpcon) {
		tcpcon_timeouts(tcpcon, secs, usecs);
		}

		/*
		* parse the response packet
		* "Preprocess" the response packet. This means to go through and
		* figure out where the TCP/IP headers are and the locations of
		* some fields, like IP address and port numbers.
		*/
		x = preprocess_frame(px, length, 1, &parsed);
		if (!x)
		continue; /* corrupt packet */
		dst = parsed.ip_dst[0]<<24 \| parsed.ip_dst[1]<<16
		ip_me = parsed.ip_dst[0]<<24 \| parsed.ip_dst[1]<<16
		\| parsed.ip_dst[2]<< 8 \| parsed.ip_dst[3]<<0;
		src = parsed.ip_src[0]<<24 \| parsed.ip_src[1]<<16
		ip_them = parsed.ip_src[0]<<24 \| parsed.ip_src[1]<<16
		\| parsed.ip_src[2]<< 8 \| parsed.ip_src[3]<<0;
		seqno = px[parsed.transport_offset+8]<<24 \| px[parsed.transport_offset+9]<<16
		\| px[parsed.transport_offset+10]<<8 \| px[parsed.transport_offset+11];
		seqno -= 1;
		seqno_them = TCP_SEQNO(px, parsed.transport_offset);
		seqno_me = TCP_ACKNO(px, parsed.transport_offset);


		/* verify: my IP address */
		if (adapter_ip != dst)
		if (adapter_ip != ip_me)
		continue;


		@@ -299,13 +377,13 @@ receive_thread(struct Masscan *masscan,
		arp_response(
		adapter_ip, adapter_mac, px, length,
		masscan->packet_buffers,
		masscan->pending_packets);
		masscan->transmit_queue);
		continue;
		}

		/* verify: TCP */
		if (parsed.found != FOUND_TCP)
		continue;
		continue; /TODO: fix for UDP-scan and ICMP-scan /

		/* verify: my port number */
		if (adapter_port != parsed.port_dst)
		@@ -322,14 +400,63 @@ receive_thread(struct Masscan *masscan,
		usecs);
		}

		/* verify: syn-cookies */
		if (syn_hash(src, parsed.port_src) != seqno) {
		LOG(1, "bad packet: ackno=0x%08x expected=0x%08x\n", seqno, syn_hash(src, parsed.port_src));
		}

		/* verify: ignore duplicates */
		if (dedup_is_duplicate(dedup, src, parsed.port_src))
		/* If recording banners, create a new "TCP Control Block (TCB)" */
		if (tcpcon) {
		struct TCP_Control_Block *tcb;

		/* does a TCB already exist for this connection? */
		tcb = tcpcon_lookup_tcb(tcpcon,
		ip_me, ip_them,
		parsed.port_dst, parsed.port_src);

		if (TCP_IS_SYNACK(px, parsed.transport_offset)) {
		if (syn_hash(ip_them, parsed.port_src) != seqno_me - 1) {
		LOG(1, "bad packet: ackno=0x%08x expected=0x%08x\n", seqno_me-1, syn_hash(ip_them, parsed.port_src));
		continue;
		}

		if (tcb == NULL) {
		tcb = tcpcon_create_tcb(tcpcon,
		ip_me, ip_them,
		parsed.port_dst,
		parsed.port_src,
		seqno_me, seqno_them+1);
		}

		tcpcon_handle(tcpcon, tcb, TCP_WHAT_SYNACK, 0, 0, secs, usecs);

		} else if (tcb) {
		/* If this is an ACK, then handle that first */
		if (TCP_IS_ACK(px, parsed.transport_offset)) {
		tcpcon_handle(tcpcon, tcb, TCP_WHAT_ACK, 0, seqno_me,
		secs, usecs);
		}

		/* If this contains payload, handle that */
		if (parsed.app_length) {
		tcpcon_handle(tcpcon, tcb, TCP_WHAT_DATA,
		px + parsed.app_offset, parsed.app_length,
		secs, usecs);
		}

		/* If this is a FIN, handle that. Note that ACK + payload + FIN
		* can come together */
		if (TCP_IS_FIN(px, parsed.transport_offset)) {
		tcpcon_handle(tcpcon, tcb, TCP_WHAT_FIN, 0, 0,
		secs, usecs);
		}

		/* If this is a RST, then we'll be closing the connection */
		if (TCP_IS_RST(px, parsed.transport_offset)) {
		tcpcon_handle(tcpcon, tcb, TCP_WHAT_RST, 0, 0,
		secs, usecs);
		}
		} else if (TCP_IS_FIN(px, parsed.transport_offset)) {
		/* TODO: we ought to send FIN-ACK in response */
		}

		}

		/* figure out the status */
		status = Port_Unknown;
		@@ -338,6 +465,15 @@ receive_thread(struct Masscan *masscan,
		if ((px[parsed.transport_offset+13] & 0x4) == 0x4)
		status = Port_Closed;

		/* verify: syn-cookies */
		if (syn_hash(ip_them, parsed.port_src) != seqno_me - 1) {
		LOG(1, "bad packet: ackno=0x%08x expected=0x%08x\n", seqno_me-1, syn_hash(ip_them, parsed.port_src));
		continue;
		}

		/* verify: ignore duplicates */
		if (dedup_is_duplicate(dedup, ip_them, parsed.port_src))
		continue;

		/*
		* XXXX
		@@ -348,7 +484,7 @@ receive_thread(struct Masscan *masscan,
		output_report(
		out,
		status,
		src,
		ip_them,
		parsed.port_src,
		px[parsed.transport_offset + 13], /* tcp flags */
		px[parsed.ip_offset + 8] /* ttl */
		@@ -499,12 +635,12 @@ main_scan(struct Masscan *masscan)
		* Allocate packet buffers for sending
		*/
		masscan->packet_buffers = rte_ring_create(256, RING_F_SP_ENQ\|RING_F_SC_DEQ);
		masscan->pending_packets = rte_ring_create(256, RING_F_SP_ENQ\|RING_F_SC_DEQ);
		masscan->transmit_queue = rte_ring_create(256, RING_F_SP_ENQ\|RING_F_SC_DEQ);
		{
		unsigned i;
		for (i=0; i<255 /TODO: why not 256???/; i++) {
		char pkt = (char)malloc(1600);
		err = rte_ring_sp_enqueue(masscan->packet_buffers, pkt);
		struct PacketBuffer p = (struct PacketBuffer )malloc(sizeof(*p));
		err = rte_ring_sp_enqueue(masscan->packet_buffers, p);
		if (err) {
		/* I dunno why but I can't queue all 256 packets, just 255 */
		LOG(0, "packet_buffers: enqueue: error %d\n", err);
		@@ -513,6 +649,8 @@ main_scan(struct Masscan *masscan)
		}




		/*
		* Start the scanning thread.
		* THIS IS WHERE THE PROGRAM STARTS SPEWING OUT PACKETS AT A HIGH
		@@ -569,6 +707,15 @@ int main(int argc, char *argv[])
		/* Set randomization seed for SYN-cookies */
		syn_set_entropy(masscan->seed);


		/* If the IP address range is very big, then require that that the
		* user apply an exclude range */
		if (rangelist_count(&masscan->targets) > 1000000000ULL
		&& rangelist_count(&masscan->exclude_ip) == 0) {
		LOG(0, "FAIL: no --exclude specified\n");
		exit(1);
		}

		/*
		* Apply excludes. People ask us not to scan them, so we maintain a list
		* of their ranges, and when doing wide scans, add the exclude list to
		@@ -630,6 +777,9 @@ int main(int argc, char *argv[])
		x += pixie_time_selftest();
		//x += xring_selftest();
		x += rte_ring_selftest();
		x += smack_selftest();
		x += banner1_selftest();



		if (x != 0) {

src/masscan.h

+7 −2

Original line number	Diff line number	Diff line
		@@ -7,9 +7,12 @@
		#include <time.h>

		#include "ranges.h"
		#include "packet-queue.h"

		struct Adapter;
		struct TcpPacket;
		struct Banner1;

		extern time_t global_now;

		enum {
		@@ -93,6 +96,7 @@ struct Masscan

		unsigned is_pfring:1;
		unsigned is_sendq:1;
		unsigned is_banners:1;

		/**
		* Wait forever for responses, instead of the default 10 seconds
		@@ -154,8 +158,9 @@ struct Masscan
		char rotate_directory[256];
		char pcap_filename[256];

		struct rte_ring *packet_buffers;
		struct rte_ring *pending_packets;
		PACKET_QUEUE *packet_buffers;
		PACKET_QUEUE *transmit_queue;

		};

src/output.c

+207 −14

Original line number	Diff line number	Diff line
		@@ -24,8 +24,10 @@
		#include "masscan.h"
		#include "string_s.h"
		#include "logger.h"
		#include "proto-banner1.h"

		#include <limits.h>
		#include <ctype.h>

		#if defined(WIN32)
		#include <Windows.h>
		@@ -37,6 +39,9 @@
		#endif

		extern unsigned control_c_pressed;



		struct Output
		{
		struct Masscan *masscan;
		@@ -47,6 +52,7 @@ struct Output
		unsigned offset;
		uint64_t open_count;
		uint64_t closed_count;
		uint64_t banner_count;
		};


		@@ -65,6 +71,14 @@ open_rotate(struct Output output, const char filename)
		unsigned is_append =masscan->nmap.append;

		#if defined(WIN32)
		/* PORTABILITY: WINDOWS
		* This bit of code deals with the fact that on Windows, fopen() opens
		* a file so that it can't be moved. This code opens it a different
		* way so that we can move it.
		*
		* NOTE: this is probably overkill, it appears that there is a better
		* API _fsopen() that does what I want without all this nonsense.
		*/
		HANDLE hFile;
		int fd;

		@@ -469,20 +483,199 @@ output_report(struct Output *out, int status, unsigned ip, unsigned port, unsign
		break;
		case Output_Binary:
		{
		struct {
		unsigned timestamp;
		unsigned ip;
		unsigned short port;
		unsigned char reason;
		unsigned char ttl;
		} foo;
		foo.timestamp = (unsigned)global_now;
		foo.ip = ip;
		foo.port = (unsigned short)port;
		foo.reason = (unsigned char)reason;
		foo.ttl = (unsigned char)ttl;

		fwrite(&foo, 1, 12, fp);
		unsigned foo[256];

		/* [TYPE] field */
		switch (status) {
		case Port_Open:
		foo[0] = 1;
		break;
		case Port_Closed:
		foo[0] = 2;
		break;
		default:
		return;
		}

		/* [LENGTH] field */
		foo[1] = 12;

		/* [TIMESTAMP] field */
		foo[2] = (unsigned char)(global_now>>24);
		foo[3] = (unsigned char)(global_now>>16);
		foo[4] = (unsigned char)(global_now>> 8);
		foo[5] = (unsigned char)(global_now>> 0);

		foo[6] = (unsigned char)(ip>>24);
		foo[7] = (unsigned char)(ip>>16);
		foo[8] = (unsigned char)(ip>> 8);
		foo[9] = (unsigned char)(ip>> 0);

		foo[10] = (unsigned char)(port>>8);
		foo[11] = (unsigned char)(port>>0);

		foo[12] = (unsigned char)reason;
		foo[13] = (unsigned char)ttl;



		fwrite(&foo, 1, 14, fp);
		}
		break;
		default:
		LOG(0, "output: ERROR: unknown format\n");
		exit(1);

		}

		}

		const char *
		proto_string(unsigned proto)
		{
		static char tmp[64];
		switch (proto) {
		case PROTO_SSH1: return "ssh";
		case PROTO_SSH2: return "ssh";
		case PROTO_HTTP: return "http";
		case PROTO_FTP1: return "ftp";
		case PROTO_FTP2: return "ftp";
		default:
		sprintf_s(tmp, sizeof(tmp), "(%u)", proto);
		return tmp;
		}
		}
		const char *
		banner_string(const unsigned char px, size_t length, char buf, size_t buf_len)
		{
		size_t i=0;
		size_t offset = 0;

		for (i=0; i<length; i++) {

		if (isprint(px[i]) && px[i] != '<' && px[i] != '>' && px[i] != '&' && px[i] != '\\') {
		if (offset + 2 < buf_len)
		buf[offset++] = px[i];
		} else {
		if (offset + 5 < buf_len) {
		buf[offset++] = '\\';
		buf[offset++] = 'x';
		buf[offset++] = "0123456789abdef"[px[i]>>4];
		buf[offset++] = "0123456789abdef"[px[i]>>0];
		}
		}
		}

		buf[offset] = '\0';

		return buf;
		}

		/***************************************************************************
		***************************************************************************/
		void
		output_report_banner(struct Output out, unsigned ip, unsigned port, unsigned proto, const unsigned char px, unsigned length)
		{
		struct Masscan *masscan = out->masscan;
		FILE *fp = out->fp;
		time_t now = time(0);

		global_now = now;


		if (masscan->nmap.format == Output_Interactive \|\| masscan->nmap.format == Output_All) {
		fprintf(stdout, "Banner on port %u/tcp on %u.%u.%u.%u: %.*s \n",
		port,
		(ip>>24)&0xFF,
		(ip>>16)&0xFF,
		(ip>> 8)&0xFF,
		(ip>> 0)&0xFF,
		length, px
		);
		}


		if (fp == NULL)
		return;

		if (now >= out->next_rotate) {
		fp = output_do_rotate(out);
		if (fp == NULL)
		return;
		}

		switch (masscan->nmap.format) {
		case Output_List:
		fprintf(fp, "%s tcp %u %u.%u.%u.%u %u %.*s\n",
		"banner",
		port,
		(ip>>24)&0xFF,
		(ip>>16)&0xFF,
		(ip>> 8)&0xFF,
		(ip>> 0)&0xFF,
		(unsigned)global_now,
		length, px
		);
		break;
		case Output_XML:
		{
		char banner_buffer[1024];
		fprintf(fp, "<host endtime=\"%u\">"
		"<address addr=\"%u.%u.%u.%u\" addrtype=\"ipv4\"/>"
		"<ports>"
		"<port protocol=\"tcp\" portid=\"%u\">"
		"<service name=\"%s\">"
		"<banner>%.*s</banner>"
		"</service>"
		"</port>"
		"</ports>"
		"</host>"
		"\r\n",
		(unsigned)global_now,
		(ip>>24)&0xFF,
		(ip>>16)&0xFF,
		(ip>> 8)&0xFF,
		(ip>> 0)&0xFF,
		port,
		proto_string(proto),
		banner_string(px, length, banner_buffer, sizeof(banner_buffer))
		);
		}
		break;
		case Output_Binary:
		{
		unsigned foo[256];

		if (length > 255 - 12)
		length = 255 - 12;

		/* [TYPE] field */
		foo[0] = 3; /banner/

		/* [LENGTH] field */
		foo[1] = length + 12;

		/* [TIMESTAMP] field */
		foo[2] = (unsigned char)(global_now>>24);
		foo[3] = (unsigned char)(global_now>>16);
		foo[4] = (unsigned char)(global_now>> 8);
		foo[5] = (unsigned char)(global_now>> 0);

		foo[6] = (unsigned char)(ip>>24);
		foo[7] = (unsigned char)(ip>>16);
		foo[8] = (unsigned char)(ip>> 8);
		foo[9] = (unsigned char)(ip>> 0);

		foo[10] = (unsigned char)(port>>8);
		foo[11] = (unsigned char)(port>>0);

		/* Banner */
		memcpy(foo, px, length);




		fwrite(&foo, 1, length+12, fp);
		}
		break;
		default: