large exlude list speed improvement (e8b985e9) · Commits · nimrod / masscan

src/main-conf.c

+15 −3

Original line number	Diff line number	Diff line
		@@ -169,6 +169,18 @@ masscan_echo(struct Masscan masscan, FILE fp)
		(range.begin>> 0)&0xFF
		);
		if (range.begin != range.end) {
		unsigned i;

		for (i=0; i<30; i++) {
		if ((range.begin&(1<<i))==0 && (range.end&(1<<i)))
		;
		else
		break;
		}
		i = 32-i;
		if ((range.begin & (0xFFFFFFFF>>i)) == ((range.end & (0xFFFFFFFF>>i)))) {
		fprintf(fp, "/%u", i);
		} else
		fprintf(fp, "-%u.%u.%u.%u",
		(range.end>>24)&0xFF,
		(range.end>>16)&0xFF,
		@@ -226,9 +238,9 @@ void ranges_from_file(struct RangeList ranges, const char filename)

		err = fopen_s(&fp, filename, "rt");
		if (err) {
		char dirname[256];
		//char dirname[256];
		perror(filename);
		fprintf(stderr, "dir = %s\n", getcwd(dirname));
		//fprintf(stderr, "dir = %s\n", getcwd(dirname));
		exit(1); /* HARD EXIT: because if it's an exclusion file, we don't
		* want to continue. We don't want ANY chance of
		* accidentally scanning somebody */
		@@ -273,7 +285,7 @@ void ranges_from_file(struct RangeList ranges, const char filename)


		/* fetch next address range */
		address[0] = c;
		address[0] = (char)c;
		i = 1;
		while (!feof(fp)) {
		c = getc(fp);

src/main.c

+6 −1

Original line number	Diff line number	Diff line
		@@ -54,6 +54,7 @@ scanning_thread(void *v)
		uint64_t seed;
		unsigned packet_trace = masscan->nmap.packet_trace;
		double timestamp_start;
		unsigned *picker;

		status_start(&status);
		throttler_start(&throttler, masscan->max_rate);
		@@ -66,6 +67,8 @@ scanning_thread(void *v)
		*/
		seed = masscan->resume.seed;

		picker = rangelist_pick2_create(&masscan->targets);

		/*
		* the main loop
		*/
		@@ -90,7 +93,7 @@ scanning_thread(void *v)
		seed = (seed * a + c) % m;

		/* Pick the IPv4 address pointed to by this index */
		ip = rangelist_pick(&masscan->targets, seed%count_ips);
		ip = rangelist_pick2(&masscan->targets, seed%count_ips, picker);
		port = rangelist_pick(&masscan->ports, seed/count_ips);

		/* Send the probe */
		@@ -586,6 +589,8 @@ int main(int argc, char *argv[])
		struct Range range = masscan->exclude_port.list[i];
		rangelist_remove_range(&masscan->ports, range.begin, range.end);
		}

		rangelist_remove_range2(&masscan->targets, range_parse_ipv4("224.0.0.0/4", 0, 0));
		}

src/pixie-threads.c

+4 −0

Original line number	Diff line number	Diff line
		@@ -10,7 +10,11 @@
		#endif

		#ifndef UNUSEDPARM
		#ifdef _MSC_VER
		#define UNUSEDPARM(x) x
		#else
		#define UNUSEDPARM(x)
		#endif
		#endif

		/****************************************************************************

src/ranges.c

+112 −8

Original line number	Diff line number	Diff line
		@@ -12,6 +12,7 @@

		#define BUCKET_COUNT 16

		#define REGRESS(x) if (!(x)) return (fprintf(stderr, "regression failed %s:%u\n", __FILE__, __LINE__)\|1)

		/***************************************************************************
		* ???
		@@ -300,7 +301,7 @@ range_parse_ipv4(const char line, unsigned inout_offset, unsigned max)
		* Handle CIDR address of the form "10.0.0.0/8"
		*/
		if (line[offset] == '/') {
		unsigned prefix = 0;
		uint64_t prefix = 0;
		uint64_t mask = 0;
		unsigned digits = 0;

		@@ -325,7 +326,7 @@ range_parse_ipv4(const char line, unsigned inout_offset, unsigned max)
		return badrange;

		/* Create the mask from the prefix */
		mask = 0xFFFFFFFF00000000UL >> prefix;
		mask = 0xFFFFFFFF00000000ULL >> prefix;

		/* Mask off any non-zero bits from the start
		* TODO print warning */
		@@ -405,6 +406,108 @@ rangelist_pick(struct RangeList *targets, uint64_t index)
		assert(!"end of list");
		return 0;
		}
		unsigned *
		rangelist_pick2_create(struct RangeList *targets)
		{
		unsigned *picker;
		unsigned i;
		unsigned total = 0;

		picker = (unsigned )malloc(targets->count sizeof(*picker));
		for (i=0; i<targets->count; i++) {
		picker[i] = total;
		total += targets->list[i].end - targets->list[i].begin + 1;
		}
		return picker;
		}
		void
		rangelist_pick2_destroy(unsigned *picker)
		{
		if (picker)
		free(picker);
		}
		unsigned
		rangelist_pick2(struct RangeList targets, uint64_t index, unsigned picker)
		{
		unsigned maxmax = targets->count;
		unsigned min = 0;
		unsigned max = targets->count;
		unsigned mid;

		for (;;) {
		mid = min + (max-min)/2;
		if (index < picker[mid]) {
		max = mid;
		continue;
		} if (index >= picker[mid]) {
		if (mid + 1 == maxmax)
		break;
		else if (index < picker[mid+1])
		break;
		else
		min = mid+1;
		}
		}

		return (unsigned)(targets->list[mid].begin + (index - picker[mid]));
		}
		int
		regress_pick2()
		{
		unsigned i;

		/*
		* Run 100 randomized regression tests
		*/
		for (i=0; i<100; i++) {
		unsigned j;
		unsigned num_targets;
		unsigned begin = 0;
		unsigned end;
		struct RangeList targets[1];
		struct RangeList duplicate[1];
		unsigned *picker;
		unsigned range;

		/* seed this test so that it's reproducible (on this platform) */
		srand(i);

		/* Create a new target list */
		memset(targets, 0, sizeof(targets[0]));

		/* fill the target list with random ranges */
		num_targets = rand()%5 + 1;
		for (j=0; j<num_targets; j++) {
		begin += rand()%10;
		end = begin + rand()%10;

		rangelist_add_range(targets, begin, end);
		}
		range = (unsigned)rangelist_count(targets);

		/* Create a "picker" */
		picker = rangelist_pick2_create(targets);

		/* Duplicate the targetlist using the picker */
		memset(duplicate, 0, sizeof(duplicate[0]));
		for (j=0; j<range; j++) {
		unsigned x;

		x = rangelist_pick2(targets, j, picker);
		rangelist_add_range(duplicate, x, x);
		}

		/* at this point, the two range lists shouild be identical */
		REGRESS(targets->count == duplicate->count);
		REGRESS(memcmp(targets->list, duplicate->list, targets->count*sizeof(targets->list[0])) == 0);

		rangelist_free(targets);
		rangelist_free(duplicate);
		}

		return 0;
		}


		/***************************************************************************
		***************************************************************************/
		@@ -450,26 +553,27 @@ ranges_selftest()
		struct Range r;
		struct RangeList task[1];

		REGRESS(regress_pick2() == 0);

		memset(task, 0, sizeof(task[0]));
		#define ASSURT(x) if (!(x)) return (fprintf(stderr, "regression failed %s:%u\n", __FILE__, __LINE__)\|1)
		#define ERROR() fprintf(stderr, "selftest: failed %s:%u\n", __FILE__, __LINE__);

		/* test for the /0 CIDR block, since we'll be using that a lot to scan the entire
		* Internet */
		r = range_parse_ipv4("0.0.0.0/0", 0, 0);
		ASSURT(r.begin == 0 && r.end == 0xFFFFFFFF);
		REGRESS(r.begin == 0 && r.end == 0xFFFFFFFF);

		r = range_parse_ipv4("0.0.0./0", 0, 0);
		ASSURT(r.begin > r.end);
		REGRESS(r.begin > r.end);

		r = range_parse_ipv4("75.748.86.91", 0, 0);
		ASSURT(r.begin > r.end);
		REGRESS(r.begin > r.end);

		r = range_parse_ipv4("23.75.345.200", 0, 0);
		ASSURT(r.begin > r.end);
		REGRESS(r.begin > r.end);

		r = range_parse_ipv4("192.1083.0.1", 0, 0);
		ASSURT(r.begin > r.end);
		REGRESS(r.begin > r.end);

		r = range_parse_ipv4("192.168.1.3", 0, 0);
		if (r.begin != 0xc0a80103 \|\| r.end != 0xc0a80103) {

src/ranges.h

+8 −0

Original line number	Diff line number	Diff line
		@@ -24,10 +24,18 @@ struct RangeList

		void rangelist_add_range(struct RangeList *task, unsigned begin, unsigned end);
		void rangelist_remove_range(struct RangeList *task, unsigned begin, unsigned end);
		void rangelist_remove_range2(struct RangeList *task, struct Range range);

		struct Range range_parse_ipv4(const char line, unsigned inout_offset, unsigned max);
		uint64_t rangelist_count(struct RangeList *targets);
		unsigned rangelist_pick(struct RangeList *targets, uint64_t i);
		void rangelist_parse_ports(struct RangeList ports, const char string);

		void rangelist_free(struct RangeList *list);


		unsigned rangelist_pick2_create(struct RangeList targets);
		void rangelist_pick2_destroy(unsigned *picker);
		unsigned rangelist_pick2(struct RangeList targets, uint64_t index, unsigned picker);

		#endif