From de78e2d59e3924238648b8fb363a1412fa9e499d Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 13 Dec 2015 17:12:43 +0000 Subject: [PATCH] Redis: move from Experimental to mainline --- doc/doc-docbook/spec.xfpt | 51 ++++++++++++++++--- doc/doc-txt/ChangeLog | 3 ++ doc/doc-txt/experimental-spec.txt | 83 ------------------------------- src/scripts/lookups-Makefile | 10 +--- src/src/EDITME | 17 +++---- src/src/config.h.defaults | 2 +- src/src/deliver.c | 2 - src/src/drtables.c | 8 +-- src/src/exim.c | 6 +-- src/src/expand.c | 20 ++++---- src/src/globals.c | 2 +- src/src/globals.h | 2 +- src/src/lookups/redis.c | 8 +-- src/src/readconf.c | 2 +- src/src/route.c | 2 - test/runtest | 2 +- test/scripts/2700-redis/REQUIRES | 2 +- 17 files changed, 83 insertions(+), 139 deletions(-) diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 13a426c00..33f35bf89 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -6624,6 +6624,14 @@ password value. For example: &(pgsql)&: The format of the query is an SQL statement that is passed to a PostgreSQL database. See section &<>&. +.next +.new +.cindex "Redis lookup type" +.cindex lookup Redis +&(redis)&: The format of the query is an SQL statement that is passed to a +Redis database. See section &<>&. +.wen + .next .cindex "sqlite lookup type" .cindex "lookup" "sqlite" @@ -7489,7 +7497,10 @@ operator is to double any quote characters within the text. .cindex "lookup" "Oracle" .cindex "InterBase lookup type" .cindex "lookup" "InterBase" -Exim can support lookups in InterBase, MySQL, Oracle, PostgreSQL, and SQLite +.cindex "Redis lookup type" +.cindex lookup Redis +Exim can support lookups in InterBase, MySQL, Oracle, PostgreSQL, Redis, +and SQLite databases. Queries for these databases contain SQL statements, so an example might be .code @@ -7516,7 +7527,7 @@ If the result of the query yields more than one row, it is all concatenated, with a newline between the data for each row. -.section "More about MySQL, PostgreSQL, Oracle, and InterBase" "SECID72" +.section "More about MySQL, PostgreSQL, Oracle, InterBase, and Redis" "SECID72" .cindex "MySQL" "lookup type" .cindex "PostgreSQL lookup type" .cindex "lookup" "MySQL" @@ -7525,13 +7536,18 @@ with a newline between the data for each row. .cindex "lookup" "Oracle" .cindex "InterBase lookup type" .cindex "lookup" "InterBase" -If any MySQL, PostgreSQL, Oracle, or InterBase lookups are used, the -&%mysql_servers%&, &%pgsql_servers%&, &%oracle_servers%&, or &%ibase_servers%& +.cindex "Redis lookup type" +.cindex lookup Redis +If any MySQL, PostgreSQL, Oracle, InterBase or Redis lookups are used, the +&%mysql_servers%&, &%pgsql_servers%&, &%oracle_servers%&, &%ibase_servers%&, +or &%redis_servers%& option (as appropriate) must be set to a colon-separated list of server information. -(For MySQL and PostgreSQL only, the global option need not be set if all +(For MySQL and PostgreSQL, the global option need not be set if all queries contain their own server information &-- see section -&<>&.) Each item in the list is a slash-separated list of four +&<>&.) +For all but Redis +each item in the list is a slash-separated list of four items: host name, database name, user name, and password. In the case of Oracle, the host name field is used for the &"service name"&, and the database name field is not used and should be empty. For example: @@ -7552,15 +7568,36 @@ a query is successfully processed. The result of a query may be that no data is found, but that is still a successful query. In other words, the list of servers provides a backup facility, not a list of different places to look. +.new +For Redis the global option need not be specified if all queries contain their +own server information &-- see section &<>&. +If specified, the option must be set to a colon-separated list of server +information. +Each item in the list is a slash-separated list of three items: +host, database number, and password. +.olist +The host is required and may be either an IPv4 address and optional +port number (separated by a colon, which needs doubling due to the +higher-level list), or a Unix socket pathname enclosed in parentheses +.next +The database number is optional; if present that number is selected in the backend +.next +The password is optional; if present it is used to authenticate to the backend +.endlist +.wen + .new The &%quote_mysql%&, &%quote_pgsql%&, and &%quote_oracle%& expansion operators convert newline, tab, carriage return, and backspace to \n, \t, \r, and \b respectively, and the characters single-quote, double-quote, and backslash itself are escaped with backslashes. + +The &%quote_redis%& expansion operator +escapes whitespace and backslash characters with a backslash. .wen .section "Specifying the server in the query" "SECTspeserque" -For MySQL and PostgreSQL lookups (but not currently for Oracle and InterBase), +For MySQL, PostgreSQL and Redis lookups (but not currently for Oracle and InterBase), it is possible to specify a list of servers with an individual query. This is done by starting the query with .display diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog index 4622fbd36..6976d4bfe 100644 --- a/doc/doc-txt/ChangeLog +++ b/doc/doc-txt/ChangeLog @@ -134,6 +134,9 @@ JH/30 Updated DANE implementation code to current from Viktor Dukhovni. JH/31 Fix bug with hosts_connection_nolog and named-lists which were wrongly cached by the daemon. +JH/32 Move Redis support from Experimental to mainline, enabled for a build + by defining LOOKUP_REDIS. The libhiredis library is required. + Exim version 4.86 ----------------- diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt index 81acfda5e..4836a7d51 100644 --- a/doc/doc-txt/experimental-spec.txt +++ b/doc/doc-txt/experimental-spec.txt @@ -771,89 +771,6 @@ b. Configure, somewhere before the DATA ACL, the control option to -Redis Lookup --------------------------------------------------------------- - -Redis is open source advanced key-value data store. This document -does not explain the fundamentals, you should read and understand how -it works by visiting the website at http://www.redis.io/. - -Redis lookup support is added via the hiredis library. Visit: - - https://github.com/redis/hiredis - -to obtain a copy, or find it in your operating systems package repository. -If building from source, this description assumes that headers will be in -/usr/local/include, and that the libraries are in /usr/local/lib. - -1. In order to build exim with Redis lookup support add - -EXPERIMENTAL_REDIS=yes - -to your Local/Makefile. (Re-)build/install exim. exim -d should show -Experimental_Redis in the line "Support for:". - -EXPERIMENTAL_REDIS=yes -LDFLAGS += -lhiredis -# CFLAGS += -I/usr/local/include -# LDFLAGS += -L/usr/local/lib - -The first line sets the feature to include the correct code, and -the second line says to link the hiredis libraries into the -exim binary. The commented out lines should be uncommented if you -built hiredis from source and installed in the default location. -Adjust the paths if you installed them elsewhere, but you do not -need to uncomment them if an rpm (or you) installed them in the -package controlled locations (/usr/include and /usr/lib). - - -2. Use the following global settings to configure Redis lookup support: - -Required: -redis_servers This option provides a list of Redis servers - and associated connection data, to be used in - conjunction with redis lookups. The option is - only available if Exim is configured with Redis - support. - -For example: - -redis_servers = 127.0.0.1/10/ - using database 10 with no password -redis_servers = 127.0.0.1//password - to make use of the default database of 0 with a password -redis_servers = 127.0.0.1// - for default database of 0 with no password - -3. Once you have the Redis servers defined you can then make use of the -experimental Redis lookup by specifying ${lookup redis{}} in a lookup query. - -4. Example usage: - -(Host List) -hostlist relay_from_ips = <\n ${lookup redis{SMEMBERS relay_from_ips}} - -Where relay_from_ips is a Redis set which contains entries such as "192.168.0.0/24" "10.0.0.0/8" and so on. -The result set is returned as -192.168.0.0/24 -10.0.0.0/8 -.. -. - -(Domain list) -domainlist virtual_domains = ${lookup redis {HGET $domain domain}} - -Where $domain is a hash which includes the key 'domain' and the value '$domain'. - -(Adding or updating an existing key) -set acl_c_spammer = ${if eq{${lookup redis{SPAMMER_SET}}}{OK}} - -Where SPAMMER_SET is a macro and it is defined as - -"SET SPAMMER " - -(Getting a value from Redis) - -set acl_c_spam_host = ${lookup redis{GET...}} - - DANE ------------------------------------------------------------ DNS-based Authentication of Named Entities, as applied diff --git a/src/scripts/lookups-Makefile b/src/scripts/lookups-Makefile index 61493c632..4e69a9a77 100755 --- a/src/scripts/lookups-Makefile +++ b/src/scripts/lookups-Makefile @@ -161,7 +161,7 @@ sed -n "1,/$tag_marker/p" < "$input" for name_mod in \ CDB DBM:dbmdb DNSDB DSEARCH IBASE LSEARCH MYSQL NIS NISPLUS ORACLE \ - PASSWD PGSQL SQLITE TESTDB WHOSON + PASSWD PGSQL REDIS SQLITE TESTDB WHOSON do emit_module_rule $name_mod done @@ -177,14 +177,6 @@ fi OBJ="${OBJ} spf.o" -# Because the variable is EXPERIMENTAL_REDIS and not LOOKUP_REDIS we -# use a different function to check for EXPERIMENTAL_* features -# requested. Don't use the SPF method with dummy functions above. -if want_experimental REDIS -then - OBJ="${OBJ} redis.o" -fi - echo "MODS = $MODS" echo "OBJ = $OBJ" diff --git a/src/src/EDITME b/src/src/EDITME index 7cdcfc93a..ab7f18588 100644 --- a/src/src/EDITME +++ b/src/src/EDITME @@ -288,6 +288,11 @@ TRANSPORT_SMTP=yes # library. # NOTE: LDAP cannot be built as a module! # +# For Redis you need to have hiredis installed on your system +# (https://github.com/redis/hiredis). +# Depending on where it is installed you may have to edit the CFLAGS +# (often += -I/usr/local/include) and LDFLAGS (-lhiredis) lines. + # If your system has pkg-config then the _INCLUDE/_LIBS setting can be # handled for you automatically by also defining the _PC variable to reference # the name of the pkg-config package, if such is available. @@ -306,6 +311,7 @@ LOOKUP_DNSDB=yes # LOOKUP_ORACLE=yes # LOOKUP_PASSWD=yes # LOOKUP_PGSQL=yes +# LOOKUP_REDIS=yes # LOOKUP_SQLITE=yes # LOOKUP_SQLITE_PC=sqlite3 # LOOKUP_WHOSON=yes @@ -357,7 +363,8 @@ PCRE_CONFIG=yes # the command for linking Exim itself, not on any auxiliary programs. You # don't need to set LOOKUP_INCLUDE if the relevant directories are already # specified in INCLUDE. The settings below are just examples; -lpq is for -# PostgreSQL, -lgds is for Interbase, -lsqlite3 is for SQLite. +# PostgreSQL, -lgds is for Interbase, -lsqlite3 is for SQLite, -lhiredis +# is for Redis. # # You do not need to use this for any lookup information added via pkg-config. @@ -480,14 +487,6 @@ EXIM_MONITOR=eximon.bin # CFLAGS += -I/usr/local/include # LDFLAGS += -lopendmarc - -# Uncomment the following line to add Redis lookup support -# You need to have hiredis installed on your system (https://github.com/redis/hiredis). -# Depending on where it is installed you may have to edit the CFLAGS and LDFLAGS lines. -# EXPERIMENTAL_REDIS=yes -# CFLAGS += -I/usr/local/include -# LDFLAGS += -lhiredis - # Uncomment the following line to enable support for checking certificate # ownership # EXPERIMENTAL_CERTNAMES=yes diff --git a/src/src/config.h.defaults b/src/src/config.h.defaults index 35c5559fb..14de083fe 100644 --- a/src/src/config.h.defaults +++ b/src/src/config.h.defaults @@ -95,6 +95,7 @@ it's a default value. */ #define LOOKUP_ORACLE #define LOOKUP_PASSWD #define LOOKUP_PGSQL +#define LOOKUP_REDIS #define LOOKUP_SQLITE #define LOOKUP_TESTDB #define LOOKUP_WHOSON @@ -177,7 +178,6 @@ it's a default value. */ #define EXPERIMENTAL_DCC #define EXPERIMENTAL_DSN_INFO #define EXPERIMENTAL_DMARC -#define EXPERIMENTAL_REDIS #define EXPERIMENTAL_SPF #define EXPERIMENTAL_SRS diff --git a/src/src/deliver.c b/src/src/deliver.c index d85f010e6..e67c06183 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -1085,9 +1085,7 @@ if (addr->message) ) && ( Ustrstr(s, "mysql") != NULL || Ustrstr(s, "pgsql") != NULL -#ifdef EXPERIMENTAL_REDIS || Ustrstr(s, "redis") != NULL -#endif || Ustrstr(s, "sqlite") != NULL || Ustrstr(s, "ldap:") != NULL || Ustrstr(s, "ldapdn:") != NULL diff --git a/src/src/drtables.c b/src/src/drtables.c index 3e978243d..ec4e72683 100644 --- a/src/src/drtables.c +++ b/src/src/drtables.c @@ -465,15 +465,15 @@ extern lookup_module_info sqlite_lookup_module_info; #ifdef EXPERIMENTAL_SPF extern lookup_module_info spf_lookup_module_info; #endif -#ifdef EXPERIMENTAL_REDIS -extern lookup_module_info redis_lookup_module_info; -#endif #if defined(LOOKUP_PGSQL) && LOOKUP_PGSQL!=2 extern lookup_module_info pgsql_lookup_module_info; #endif #if defined(LOOKUP_PASSWD) && LOOKUP_PASSWD!=2 extern lookup_module_info passwd_lookup_module_info; #endif +#if defined(LOOKUP_REDIS) && LOOKUP_REDIS!=2 +extern lookup_module_info redis_lookup_module_info; +#endif #if defined(LOOKUP_ORACLE) && LOOKUP_ORACLE!=2 extern lookup_module_info oracle_lookup_module_info; #endif @@ -576,7 +576,7 @@ void init_lookup_list(void) addlookupmodule(NULL, &pgsql_lookup_module_info); #endif -#ifdef EXPERIMENTAL_REDIS +#if defined(LOOKUP_REDIS) && LOOKUP_REDIS!=2 addlookupmodule(NULL, &redis_lookup_module_info); #endif diff --git a/src/src/exim.c b/src/src/exim.c index f8cfda8a8..28617a510 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -862,9 +862,6 @@ fprintf(f, "Support for:"); #ifdef EXPERIMENTAL_DSN_INFO fprintf(f, " Experimental_DSN_info"); #endif -#ifdef EXPERIMENTAL_REDIS - fprintf(f, " Experimental_Redis"); -#endif fprintf(f, "\n"); fprintf(f, "Lookups (built-in):"); @@ -907,6 +904,9 @@ fprintf(f, "Lookups (built-in):"); #if defined(LOOKUP_PGSQL) && LOOKUP_PGSQL!=2 fprintf(f, " pgsql"); #endif +#if defined(LOOKUP_REDIS) && LOOKUP_REDIS!=2 + fprintf(f, " redis"); +#endif #if defined(LOOKUP_SQLITE) && LOOKUP_SQLITE!=2 fprintf(f, " sqlite"); #endif diff --git a/src/src/expand.c b/src/src/expand.c index f09271782..a51dac570 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -7482,22 +7482,22 @@ for (i = 1; i < argc; i++) if (Ustrspn(argv[i], "abcdefghijklmnopqrtsuvwxyz0123456789-.:/") == Ustrlen(argv[i])) { - #ifdef LOOKUP_LDAP +#ifdef LOOKUP_LDAP eldap_default_servers = argv[i]; - #endif - #ifdef LOOKUP_MYSQL +#endif +#ifdef LOOKUP_MYSQL mysql_servers = argv[i]; - #endif - #ifdef LOOKUP_PGSQL +#endif +#ifdef LOOKUP_PGSQL pgsql_servers = argv[i]; - #endif - #ifdef EXPERIMENTAL_REDIS +#endif +#ifdef LOOKUP_REDIS redis_servers = argv[i]; - #endif +#endif } - #ifdef EXIM_PERL +#ifdef EXIM_PERL else opt_perl_startup = argv[i]; - #endif +#endif } printf("Testing string expansion: debug_level = %d\n\n", debug_level); diff --git a/src/src/globals.c b/src/src/globals.c index 0edb16962..14a821a19 100644 --- a/src/src/globals.c +++ b/src/src/globals.c @@ -83,7 +83,7 @@ uschar *oracle_servers = NULL; uschar *pgsql_servers = NULL; #endif -#ifdef EXPERIMENTAL_REDIS +#ifdef LOOKUP_REDIS uschar *redis_servers = NULL; #endif diff --git a/src/src/globals.h b/src/src/globals.h index eb85eefa9..9ae78a920 100644 --- a/src/src/globals.h +++ b/src/src/globals.h @@ -62,7 +62,7 @@ extern uschar *oracle_servers; /* List of servers and connect info */ extern uschar *pgsql_servers; /* List of servers and connect info */ #endif -#ifdef EXPERIMENTAL_REDIS +#ifdef LOOKUP_REDIS extern uschar *redis_servers; /* List of servers and connect info */ #endif diff --git a/src/src/lookups/redis.c b/src/src/lookups/redis.c index 5f677ff06..7bab34971 100644 --- a/src/src/lookups/redis.c +++ b/src/src/lookups/redis.c @@ -7,7 +7,7 @@ #include "../exim.h" -#ifdef EXPERIMENTAL_REDIS +#ifdef LOOKUP_REDIS #include "lf_functions.h" @@ -118,7 +118,7 @@ if (sdata[2][0] == 0) sdata[2] = NULL; /* See if we have a cached connection to the server */ -for (cn = redis_connections; cn != NULL; cn = cn->next) +for (cn = redis_connections; cn; cn = cn->next) if (Ustrcmp(cn->server, server_copy) == 0) { redis_handle = cn->handle; @@ -458,11 +458,11 @@ static lookup_info redis_lookup_info = { }; #ifdef DYNLOOKUP -#define redis_lookup_module_info _lookup_module_info +# define redis_lookup_module_info _lookup_module_info #endif /* DYNLOOKUP */ static lookup_info *_lookup_list[] = { &redis_lookup_info }; lookup_module_info redis_lookup_module_info = { LOOKUP_MODULE_INFO_MAGIC, _lookup_list, 1 }; -#endif /* EXPERIMENTAL_REDIS */ +#endif /* LOOKUP_REDIS */ /* End of lookups/redis.c */ diff --git a/src/src/readconf.c b/src/src/readconf.c index f2bf23bbb..6f97dad69 100644 --- a/src/src/readconf.c +++ b/src/src/readconf.c @@ -375,7 +375,7 @@ static optionlist optionlist_config[] = { { "recipient_unqualified_hosts", opt_stringptr, &recipient_unqualified_hosts }, { "recipients_max", opt_int, &recipients_max }, { "recipients_max_reject", opt_bool, &recipients_max_reject }, -#ifdef EXPERIMENTAL_REDIS +#ifdef LOOKUP_REDIS { "redis_servers", opt_stringptr, &redis_servers }, #endif { "remote_max_parallel", opt_int, &remote_max_parallel }, diff --git a/src/src/route.c b/src/src/route.c index 0d96e2365..b289b0f34 100644 --- a/src/src/route.c +++ b/src/src/route.c @@ -1904,9 +1904,7 @@ if ( yield == DEFER ) && ( Ustrstr(addr->message, "mysql") != NULL || Ustrstr(addr->message, "pgsql") != NULL -#ifdef EXPERIMENTAL_REDIS || Ustrstr(addr->message, "redis") != NULL -#endif || Ustrstr(addr->message, "sqlite") != NULL || Ustrstr(addr->message, "ldap:") != NULL || Ustrstr(addr->message, "ldapdn:") != NULL diff --git a/test/runtest b/test/runtest index 724ccd9ed..88c26e273 100755 --- a/test/runtest +++ b/test/runtest @@ -2857,7 +2857,7 @@ if (defined $parm_support{'Content_Scanning'}) ################################################## # Check for redis # ################################################## -if (defined $parm_support{'Experimental_Redis'}) +if (defined $parm_lookups{'redis'}) { if (system("redis-server -v 2>/dev/null >/dev/null") == 0) { diff --git a/test/scripts/2700-redis/REQUIRES b/test/scripts/2700-redis/REQUIRES index 9c48ef88e..55fbcfdcc 100644 --- a/test/scripts/2700-redis/REQUIRES +++ b/test/scripts/2700-redis/REQUIRES @@ -1,2 +1,2 @@ -support Experimental_Redis +lookup redis running redis -- 2.25.1