shards (a54f56aa) · Commits · nimrod / masscan

src/main-conf.c

+26 −11

Original line number	Diff line number	Diff line
		@@ -98,6 +98,7 @@ masscan_echo(struct Masscan masscan, FILE fp)
		fprintf(fp, "rate = %10.2f\n", masscan->max_rate);
		fprintf(fp, "randomize-hosts = true\n");
		fprintf(fp, "seed = %llu\n", masscan->seed);
		fprintf(fp, "shard = %u/%u\n", masscan->shard.one, masscan->shard.of);


		fprintf(fp, "# ADAPTER SETTINGS\n");
		@@ -220,9 +221,6 @@ masscan_save_state(struct Masscan *masscan)
		fprintf(fp, "\n# resume information\n");
		fprintf(fp, "resume-seed = %llu\n", masscan->resume.seed);
		fprintf(fp, "resume-index = %llu\n", masscan->resume.index);
		fprintf(fp, "resume-range = %llu\n", masscan->lcg.m);
		fprintf(fp, "resume-lcg-a = %llu\n", masscan->lcg.a);
		fprintf(fp, "resume-lcg-c = %llu\n", masscan->lcg.c);

		masscan_echo(masscan, fp);

		@@ -717,14 +715,6 @@ masscan_set_parameter(struct Masscan *masscan,
		masscan->resume.seed = parseInt(value);
		} else if (EQUALS("resume-index", name)) {
		masscan->resume.index = parseInt(value);
		} else if (EQUALS("resume-range", name)) {
		masscan->lcg.m = parseInt(value);
		} else if (EQUALS("resume-lcg-m", name)) {
		masscan->lcg.m = parseInt(value);
		} else if (EQUALS("resume-lcg-a", name)) {
		masscan->lcg.a = parseInt(value);
		} else if (EQUALS("resume-lcg-c", name)) {
		masscan->lcg.c = parseInt(value);
		} else if (EQUALS("retries", name)) {
		unsigned x = strtoul(value, 0, 0);
		if (x >= 1000) {
		@@ -769,6 +759,31 @@ masscan_set_parameter(struct Masscan *masscan,
		return;
		} else if (EQUALS("source-port", name) \|\| EQUALS("sourceport", name)) {
		masscan_set_parameter(masscan, "adapter-port", value);
		} else if (EQUALS("shard", name)) {
		unsigned one = 0;
		unsigned of = 0;

		while (isdigit(*value))
		one = one10 + ((value++)) - '0';
		while (ispunct(*value))
		value++;
		while (isdigit(*value))
		of = of10 + ((value++)) - '0';

		if (one < 1) {
		LOG(0, "FAIL: shard index can't be zero\n");
		LOG(0, "hint it goes like 1/4 2/4 3/4 4/4\n");
		exit(1);
		}
		if (one > of) {
		LOG(0, "FAIL: shard spec is wrong\n");
		LOG(0, "hint it goes like 1/4 2/4 3/4 4/4\n");
		exit(1);
		}

		masscan->shard.one = one;
		masscan->shard.of = of;

		} else if (EQUALS("no-stylesheet", name)) {
		masscan->nmap.stylesheet[0] = '\0';
		} else if (EQUALS("stylesheet", name)) {

src/main.c

+120 −88

Original line number	Diff line number	Diff line
		@@ -10,6 +10,7 @@
		*/
		#include "masscan.h"

		#include "rand-blackrock.h" /* the BlackRock shuffling func */
		#include "rand-lcg.h" /* the LCG randomization func */
		#include "tcpkt.h" /* packet template, that we use to send */
		#include "rawsock.h" /* api on top of Linux, Windows, Mac OS X*/
		@@ -55,7 +56,9 @@ void
		flush_packets(struct Masscan masscan, struct Throttler throttler, uint64_t *packets_sent)
		{
		uint64_t batch_size;
		unsigned is_queue_empty = 0;

		while (!is_queue_empty) {
		/*
		* Only send a few packets at a time, throttled according to the max
		* --rate set by the usser
		@@ -75,8 +78,10 @@ flush_packets(struct Masscan masscan, struct Throttler throttler, uint64_t *pa
		* an ACK or an HTTP request
		*/
		err = rte_ring_sc_dequeue(masscan->transmit_queue, (void**)&p);
		if (err)
		if (err) {
		is_queue_empty = 1;
		break; /* queue is empty, nothing to send */
		}

		/*
		* Actually send the packet
		@@ -103,6 +108,7 @@ flush_packets(struct Masscan masscan, struct Throttler throttler, uint64_t *pa
		(*packets_sent)++;
		}
		}
		}


		/***************************************************************************
		@@ -116,15 +122,19 @@ static void
		transmit_thread(void v) /aka. scanning_thread() */
		{
		uint64_t i;
		uint64_t start;
		uint64_t end;
		struct Masscan masscan = (struct Masscan )v;
		uint64_t a = masscan->lcg.a;
		uint64_t c = masscan->lcg.c;
		uint64_t m = masscan->lcg.m;
		uint64_t seed;
		unsigned retries = masscan->retries;
		unsigned rate = (unsigned)masscan->max_rate;
		unsigned r = retries + 1;
		uint64_t range;
		struct BlackRock blackrock;
		uint64_t count_ips = rangelist_count(&masscan->targets);
		struct Status status;
		struct Throttler throttler;
		struct TcpPacket *pkt_template = masscan->pkt_template;
		uint64_t seed;
		unsigned packet_trace = masscan->nmap.packet_trace;
		double timestamp_start;
		unsigned *picker;
		@@ -133,6 +143,30 @@ transmit_thread(void v) /aka. scanning_thread() */

		LOG(1, "xmit: starting transmit thread...\n");

		/* Create the shuffler/randomizer */
		range = rangelist_count(&masscan->targets)
		* rangelist_count(&masscan->ports);
		blackrock_init(&blackrock, range);

		/* Seed */
		seed = masscan->seed;
		if (seed == 0 && masscan->shard.one == 1 && masscan->shard.of == 1)
		; //seed = time(0) % range;

		/* start/end */
		if (masscan->resume.index != 0)
		start = masscan->resume.index;
		else
		start = (masscan->shard.one-1) * (range / masscan->shard.of);

		if (masscan->shard.of == 1)
		end = range;
		else
		end = masscan->shard.one * (range / masscan->shard.of);

		end += retries * rate;


		/* "STATUS" is once-per-second <stderr> notification to the command
		* line as to what's going on */
		status_start(&status);
		@@ -146,8 +180,8 @@ transmit_thread(void v) /aka. scanning_thread() */
		timestamp_start = 1.0 * pixie_gettime() / 1000000.0;


		/* Seed the LCG for randomizing the scan*/
		seed = masscan->resume.seed;
		/* TODO: this feature useless right now*/
		//seed = masscan->resume.seed;

		/* Optimize target selection so it's a quick binary search instead
		* of walking large memory tables */
		@@ -157,7 +191,7 @@ transmit_thread(void v) /aka. scanning_thread() */
		* the main loop
		* -----------------*/
		LOG(3, "xmit: starting main loop\n");
		for (i=masscan->resume.index; i<masscan->lcg.m; ) {
		for (i=start; i<end; ) {
		uint64_t batch_size;

		/*
		@@ -168,48 +202,71 @@ transmit_thread(void v) /aka. scanning_thread() */
		*/
		batch_size = throttler_next_batch(&throttler, packets_sent);
		packets_sent += batch_size;
		while (batch_size && i < m) {
		while (batch_size && i < end) {
		uint64_t xXx;
		unsigned ip;
		unsigned port;

		batch_size--;

		/* randomize the index. THIS IS WHERE RANDOMIZATION HAPPENS
		* index = lcg_rand(index, a, c, m); */
		seed = (seed * a + c) % m;

		/* Pick the IPv4 address pointed to by this index */
		ip = rangelist_pick2(&masscan->targets, seed%count_ips, picker);
		port = rangelist_pick(&masscan->ports, seed/count_ips);
		/*
		* RANDOMIZE THE TARGET:
		* This is kinda a tricky bit that picks a random IP and port
		* number in order to scan. We monotonically increment the
		* index 'i' from [0..range]. We then shuffle (randomly transmog)
		* that index into some other, but unique/1-to-1, number in the
		* same range. That way we visit all targets, but in a random
		* order. Then, once we've shuffled the index, we "pick" the
		* the IP address and port that the index refers to.
		*/
		xXx = blackrock_shuffle(&blackrock, (i + (r--) * rate + seed) % range);
		ip = rangelist_pick2(&masscan->targets, xXx % count_ips, picker);
		port = rangelist_pick(&masscan->ports, xXx / count_ips);

		/* Print packet if debugging */
		if (packet_trace)
		tcpkt_trace(pkt_template, ip, port, timestamp_start);

		/* Send the probe */
		/*
		* SEND THE PROBE
		* This is sorta the entire point of the program, but little
		* exciting happens here. The thing to note that this may
		* be a "raw" transmit that bypasses the kernel, meaning
		* we can call this function millions of times a second.
		*/
		rawsock_send_probe(
		adapter,
		ip,
		port,
		syn_hash(ip, port),
		!batch_size, /* flush transmit queue on last packet */
		!batch_size, /* flush queue on last packet in batch */
		pkt_template
		);
		batch_size--;


		/*
		* SEQUENTIALLY INCREMENT THROUGH THE RANGE
		* Yea, I know this is a puny 'i++' here, but it's a core feature
		* of the system that is linearly increments through the range,
		* but produces from that a shuffled sequence of targets (as
		* described above). Because we are linearly incrementing this
		* number, we can do lots of creative stuff, like doing clever
		* retransmits and sharding.
		*/
		if (r == 0) {
		i++;

		r = retries + 1;
		}

		/*
		* update screen about once per second with statistics,
		* namely packets/second.
		*/
		if ((i & status.timer) == status.timer)
		status_print(&status, i, m);
		status_print(&status, i-start, end-start);

		} /* end of batch */

		/* Transmit packets from other thread */
		/* Transmit packets from other thread. */
		flush_packets(masscan, &throttler, &packets_sent);

		/* If the user pressed <ctrl-c>, then we need to exit. but, in case
		@@ -236,7 +293,7 @@ transmit_thread(void v) /aka. scanning_thread() */
		unsigned j;
		for (j=0; j<masscan->wait && !control_c_pressed; j++) {
		unsigned k;
		status_print(&status, i++, m);
		status_print(&status, i++ - start, end-start);

		for (k=0; k<1000; k++) {
		/* Transmit packets from other thread */
		@@ -612,34 +669,6 @@ main_scan(struct Masscan *masscan)
		LOG(0, "Scanning %u hosts [%u port%s/host]\n",
		(unsigned)count_ips, (unsigned)count_ports, (count_ports==1)?"":"s");

		/*
		* Initialize LCG translator
		*
		* This can take a couple seconds on a slow CPU. We have to find all the
		* primes out to 2^24 when doing large ranges.
		*/
		if (masscan->resume.index && masscan->resume.seed && masscan->lcg.m
		&& masscan->lcg.a && masscan->lcg.c) {
		if (masscan->lcg.m != count_ips * count_ports) {
		LOG(0, "FAIL: corrupt resume data\n");
		exit(1);
		} else
		LOG(0, "resuming scan...\n");
		} else {
		masscan->lcg.m = count_ips * count_ports;
		lcg_calculate_constants(
		masscan->lcg.m,
		&masscan->lcg.a,
		&masscan->lcg.c,
		0);
		LOG(2, "lcg-constants = a(%llu) c(%llu) m(%llu)\n",
		masscan->lcg.a,
		masscan->lcg.c,
		masscan->lcg.m
		);
		masscan->resume.seed = time(0) % masscan->lcg.m;
		masscan->resume.index = 0;
		}

		/*
		* trap <ctrl-c> to pause
		@@ -719,6 +748,8 @@ int main(int argc, char *argv[])
		masscan->wait = 10; /* how long to wait for responses when done */
		masscan->max_rate = 100.0; /* max rate = hundred packets-per-second */
		masscan->adapter_port = 0x10000; /* value not set */
		masscan->shard.one = 1;
		masscan->shard.of = 1;
		strcpy_s( masscan->rotate_directory,
		sizeof(masscan->rotate_directory),
		".");
		@@ -794,6 +825,7 @@ int main(int argc, char *argv[])
		*/
		{
		int x = 0;
		x += blackrock_selftest();
		x += rawsock_selftest();
		x += randlcg_selftest();
		x += tcpkt_selftest();

src/masscan.h

+5 −5

Original line number	Diff line number	Diff line
		@@ -78,11 +78,6 @@ struct Masscan
		struct RangeList exclude_port;


		struct LCGParms {
		uint64_t m; /* LCG modulus aka. the IP address range size */
		uint64_t a; /* LCG multiplier */
		uint64_t c; /* LCG increment */
		} lcg;

		/**
		* Maximum rate, in packets-per-second (--rate parameter)
		@@ -109,6 +104,11 @@ struct Masscan
		uint64_t index;
		} resume;

		struct {
		unsigned one;
		unsigned of;
		} shard;

		/**
		* The packet template we are current using
		*/

src/proto-tcp.c

+2 −0

Original line number	Diff line number	Diff line
		@@ -335,6 +335,7 @@ tcpcon_send_packet(
		for (err=1; err; ) {
		err = rte_ring_sc_dequeue(tcpcon->packet_buffers, (void**)&response);
		if (err != 0) {
		LOG(0, "packet buffers empty (should be impossible)\n");
		pixie_usleep(100); /* no packet available */
		}
		}
		@@ -368,6 +369,7 @@ tcpcon_send_packet(
		for (err=1; err; ) {
		err = rte_ring_sp_enqueue(tcpcon->transmit_queue, response);
		if (err != 0) {
		LOG(0, "transmit queue full (should be impossible)\n");
		pixie_usleep(100); /* no space available */
		}
		}

src/rand-blackrock.c

0 → 100644

+211 −0

Original line number	Diff line number	Diff line
		/*
		BlackRock cipher

		This is a randomization/reshuffling function based on a crypto
		"Feistal network" as describ ed in the paper:

		'Ciphers with Arbitrary Finite Domains'
		by John Black and Phillip Rogaway
		http://www.cs.ucdavis.edu/~rogaway/papers/subset.pdf

		This is a crypto-like construction that encrypts an arbitrary sized
		range. Given a number in the range [0..9999], it'll produce a mapping
		to a distinct different number in the same range (and back again).
		In other words, it randomizes the order of numbers in a sequence.

		For example, it can be used to randomize the sequence [0..9]:

		0 -> 6
		1 -> 4
		2 -> 8
		3 -> 1
		4 -> 9
		5 -> 3
		6 -> 0
		7 -> 5
		8 -> 2
		9 -> 7

		As you can see on the right hand side, the numbers are in random
		order, and they don't repeaet.

		This is create for port scanning. We can take an index variable
		and increment it during a scan, then use this function to
		randomize it, yet be assured that we've probed every IP and port
		within the range.

		The cryptographic strength of this construction depends upon the
		number of rounds, and the exact nature of the inner "F()" function.
		Because it's a Feistal network, that "F()" function can be almost
		anything.

		We don't care about cryptographic strength, just speed, so we are
		using a trivial F() function.

		This is a class of "format-preserving encryption". There are
		probably better constructions than what I'm using.
		*/

		#include <stdint.h>
		#include <string.h>
		#include <stdlib.h>
		#include <stdio.h>
		#include <math.h>
		#include <ctype.h>

		#if defined(_MSC_VER)
		#define inline _inline
		#endif

		struct BlackRock {
		uint64_t range;
		uint64_t a;
		uint64_t b;
		unsigned rounds;
		};


		/***************************************************************************
		***************************************************************************/
		void
		blackrock_init(struct BlackRock *br, uint64_t range)
		{
		double foo = sqrt(range * 1.0);

		br->range = range;
		br->a = (uint64_t)(foo - 1);
		br->b = (uint64_t)(foo + 1);

		while (br->a * br->b <= range)
		br->b++;

		br->rounds = 3;
		}


		/***************************************************************************
		***************************************************************************/
		uint64_t
		F(uint64_t j, uint64_t R)
		{
		static const uint64_t primes[] = {
		961752031, 982324657, 15485843 };

		R = (R << (R&0x4)) + R;

		/* some random and meaningless function */
		return (((primes[j] * R + 25ULL) ^ R) + j);
		}


		/***************************************************************************
		*
		* NOTE:
		* the names in this function are cryptic in order to match as closely
		* as possible the pseudocode in the following paper:
		* http://www.cs.ucdavis.edu/~rogaway/papers/subset.pdf
		***************************************************************************/
		static inline uint64_t
		fe(unsigned r, uint64_t a, uint64_t b, uint64_t m)
		{
		uint64_t L, R;
		unsigned j;
		uint64_t tmp;

		L = m % a;
		R = m / a;

		for (j=1; j<=r; j++) {
		if (j & 1) {
		tmp = (L + F(j, R)) % a;
		} else {
		tmp = (L + F(j, R)) % b;
		}
		L = R;
		R = tmp;
		}
		if (r & 1) {
		return a * L + R;
		} else {
		return a * R + L;
		}
		}

		/***************************************************************************
		***************************************************************************/
		uint64_t
		blackrock_shuffle(const struct BlackRock *br, uint64_t m)
		{
		uint64_t c;

		c = fe(br->rounds, br->a, br->b, m);
		while (c >= br->range)
		c = fe(br->rounds, br->a, br->b, c);

		return c;
		}

		/***************************************************************************
		***************************************************************************/
		static unsigned
		blackrock_verify(struct BlackRock *br, uint64_t max)
		{
		unsigned char *list;
		uint64_t i;
		unsigned is_success = 1;
		uint64_t range = br->range;

		/* Allocate a list of 1-byte counters */
		list = (unsigned char *)malloc((size_t)((range<max)?range:max));
		memset(list, 0, (size_t)((range<max)?range:max));

		/* For all numbers in the range, verify increment the counter for the
		* the output. */
		for (i=0; i<range; i++) {
		uint64_t x = blackrock_shuffle(br, i);
		if (x < max)
		list[x]++;
		}

		/* Now check the output to make sure that every counter is set exactly
		* to the value of '1'. */
		for (i=0; i<max && i<range; i++) {
		if (list[i] != 1)
		is_success = 0;
		}

		free(list);

		return is_success;
		}

		/***************************************************************************
		***************************************************************************/
		int
		blackrock_selftest()
		{
		unsigned i;
		int is_success = 0;
		uint64_t range;


		range = 3015 * 3;

		for (i=0; i<5; i++) {
		struct BlackRock br;

		range += 10 + i;
		range *= 2;

		blackrock_init(&br, range);

		is_success = blackrock_verify(&br, range);

		if (!is_success) {
		fprintf(stderr, "BLACKROCK: randomization failed\n");
		return 1; /fail/
		}
		}

		return 0; /success/
		}