parallel http requests program

General OpenMP discussion
Forum rules
The OpenMP Forums are now closed to new posts. Please visit Stack Overflow if you are in need of help: https://stackoverflow.com/questions/tagged/openmp
Locked
alesawe
Posts: 1
Joined: Sat Jun 06, 2020 3:49 am

parallel http requests program

Post by alesawe »

Hi OpenMP community,

I'm trying to learn OpenMP by coding a program that requests a large wordlist of hosts and to test of they are listing on http or https I've the OpenMP version of the program is faster than a normal concurrent program by 2x speed. I'm trying to make my program faster any idea is appreciated.

Code: Select all

#include <curl/curl.h>
#include <iostream>
#include <string>
#include <fstream>
#include <omp.h>


#define NUM_THREADS 40

using namespace std;

size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp)
{
   return size * nmemb;
}


int main(int argc, char const *argv[] )
{
  


	if (argc < 2) cout << "Usage httplive <path to subdomains>" << endl;

	ifstream infile(argv[1]);
	string line;
	omp_set_num_threads(NUM_THREADS/2);
	ofstream out_file;
	out_file.open("out.txt");

	#pragma omp parallel
	{


		for (string line ; getline(infile, line); ) {

			CURL *curl = curl_easy_init();
			line = "http://" + line ;
			curl_easy_setopt(curl, CURLOPT_URL, const_cast<char*>(line.c_str()));
			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
			curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 0L); 
			curl_easy_setopt(curl, CURLOPT_TCP_NODELAY, 0L);
			curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3L);     
			curl_easy_setopt(curl, CURLOPT_COOKIEFILE, "");
			curl_easy_setopt(curl, CURLOPT_COOKIE, "");
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
			curl_easy_setopt(curl, CURLOPT_VERBOSE, 0L);
			auto res = curl_easy_perform(curl);


		if (res == CURLE_OK ) out_file << line << "\n";

		}
	}

	omp_set_num_threads(NUM_THREADS/2);

	ifstream infile2(argv[1]);
	
	#pragma omp parallel
	{
	

		for (string line ; getline(infile2, line); ) {

			CURL *curl = curl_easy_init();
			line = "https://" + line ;
			curl_easy_setopt(curl, CURLOPT_URL, const_cast<char*>(line.c_str()));
			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
			curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 0L); 
			curl_easy_setopt(curl, CURLOPT_TCP_NODELAY, 0L);
			curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3L);     
			curl_easy_setopt(curl, CURLOPT_COOKIEFILE, "");
			curl_easy_setopt(curl, CURLOPT_COOKIE, "");
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
			curl_easy_setopt(curl, CURLOPT_VERBOSE, 0L);
			auto res = curl_easy_perform(curl);
		
		if (res == CURLE_OK ) out_file << line << "\n";

		}
	}

	out_file.close();
	infile.close();
	infile2.close();
	return 0;
}
Regards,
Alesawe

MarkB
Posts: 808
Joined: Thu Jan 08, 2009 10:12 am
Location: EPCC, University of Edinburgh

Re: parallel http requests program

Post by MarkB »

This is a rather unusual use case for OpenMP - normally it is used for compute intensive tasks. I think the performance of this code will be be very dependent on your I/O and network bandwidth.

Locked