diff --git a/HyperMake b/HyperMake index b36645c50..fbbf3eddd 100644 --- a/HyperMake +++ b/HyperMake @@ -50,6 +50,24 @@ targets: cmds: - ./support/scripts/pack.sh + docker-lightwave-base: + description: build lightwave-base docker image + build: ./support/docker/base + image: 'vmware/lightwave-base:1.0.0' + + docker-lightwave-pre: + description: Prepare for container build + cmds: + - ./support/scripts/prep-container-build.sh + always: true + + docker-lightwave: + description: build lightwave docker image + build: ./build/docker + image: 'vmware/lightwave-sts' + after: + - docker-lightwave-pre + clean: description: Cleanup always: true @@ -61,4 +79,4 @@ settings: - build - pack docker: - image: 'vmware/lightwave-toolchain-photon:0.0.1' + image: 'vmware/lightwave-toolchain-photon:0.0.3' diff --git a/build/package/rpm/lightwave.spec b/build/package/rpm/lightwave.spec index f118d671f..dd510bf55 100644 --- a/build/package/rpm/lightwave.spec +++ b/build/package/rpm/lightwave.spec @@ -8,8 +8,8 @@ License: VMware URL: http://www.vmware.com BuildArch: x86_64 -Requires: openssl >= 1.0.2, coreutils >= 8.22, cyrus-sasl >= 2.1, likewise-open >= 6.2.11, gawk >= 4.1.3, boost = 1.60.0, lightwave-server = %{_version}, lightwave-client = %{_version} -BuildRequires: openssl-devel >= 1.0.2, coreutils >= 8.22, likewise-open-devel >= 6.2.11, python2-devel >= 2.7.8, boost-devel = 1.60.0 +Requires: openssl >= 1.0.2, coreutils >= 8.22, cyrus-sasl >= 2.1, c-rest-engine = 1.1, likewise-open >= 6.2.11, gawk >= 4.1.3, boost = 1.60.0, lightwave-server = %{_version}, lightwave-client = %{_version} +BuildRequires: openssl-devel >= 1.0.2, coreutils >= 8.22, likewise-open-devel >= 6.2.11, python2-devel >= 2.7.8, boost-devel = 1.60.0, c-rest-engine-devel = 1.1 %if 0%{?fedora} >= 21 Requires: java-1.8.0-openjdk >= 1.8.0.131, krb5-libs >= 1.14, sqlite >= 3.14, tomcat >= 8.5.16, apache-commons-daemon >= 1.0.15, apache-commons-daemon-jsvc >= 1.0.15 @@ -1032,7 +1032,6 @@ Lightwave POST service %{_bindir}/vdcupgrade %{_bindir}/vmkdc_admin %{_bindir}/vdcmetric -%{_bindir}/vdcschema %{_bindir}/vmdir_upgrade.sh %{_bindir}/vdcresetMachineActCred @@ -1077,6 +1076,8 @@ Lightwave POST service %{_bindir}/vmdns-cli %{_bindir}/vdcaclmgr %{_bindir}/vdcpromo +%{_bindir}/vdcschema +%{_bindir}/postschema %{_bindir}/vecs-cli %{_lib64dir}/libkrb5crypto.so* %{_lib64dir}/libcsrp.so* @@ -1156,7 +1157,6 @@ Lightwave POST service %{_bindir}/postadmintool %{_bindir}/postaclmgr -%{_bindir}/postschema %{_bindir}/post-cli %{_lib64dir}/sasl2/libsaslpostdb.so* @@ -1172,6 +1172,7 @@ Lightwave POST service %{_configdir}/lw-firewall-post.json %config %attr(750, root, root) %{_datadir}/config/post-demote-deads.sh +%config %attr(750, root, root) %{_datadir}/config/refresh-resolve-conf.sh %files devel diff --git a/lwraft/client/client.c b/lwraft/client/client.c index 17ae87196..13b33d709 100644 --- a/lwraft/client/client.c +++ b/lwraft/client/client.c @@ -125,10 +125,12 @@ VmDirRefreshActPassword( BAIL_ON_VMDIR_ERROR(dwError); } - dwError = VmDirSafeLDAPBind( &pLD, - pszHost, - pszActUPN, - pszActPassword); + dwError = VmDirSafeLDAPBindExt1( + &pLD, + pszHost, + pszActUPN, + pszActPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirDomainNameToDN( pszDomain, &pszDomainDN); @@ -2415,10 +2417,12 @@ _VmDirModDcPassword( DWORD dwError = 0; LDAP* pLD = NULL; - dwError = VmDirSafeLDAPBind(&pLD, - pszHostName, - pszUPN, - pszPassword); + dwError = VmDirSafeLDAPBindExt1( + &pLD, + pszHostName, + pszUPN, + pszPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirLdapModReplaceAttribute(pLD, diff --git a/lwraft/client/defines.h b/lwraft/client/defines.h index 4738c37bd..4e7bbe5af 100644 --- a/lwraft/client/defines.h +++ b/lwraft/client/defines.h @@ -322,6 +322,8 @@ the buffer size will always be adequate. "Invalid ACE"}, \ {VMDIR_ERROR_ACE_NOT_FOUND, \ "ACE not found"}, \ + {VMDIR_ERROR_NO_LEADER, \ + "No leader"}, \ }; #define VMDIR_RPC_ERROR_TABLE_INITIALIZER \ diff --git a/lwraft/client/ldaputil.c b/lwraft/client/ldaputil.c index 8d9948a82..5efb588ef 100644 --- a/lwraft/client/ldaputil.c +++ b/lwraft/client/ldaputil.c @@ -542,10 +542,12 @@ VmDirConnectLDAPServer( dwError = VmDirAllocateStringPrintf(&pszUPN, "%s@%s", pszUserName, pszDomain); BAIL_ON_VMDIR_ERROR(dwError); - dwError = VmDirSafeLDAPBind( &pLocalLd, - pszHostName, - pszUPN, - pszPassword); + dwError = VmDirSafeLDAPBindExt1( + &pLocalLd, + pszHostName, + pszUPN, + pszPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); *ppLd = pLocalLd; diff --git a/lwraft/client/repadmin.c b/lwraft/client/repadmin.c index a1a1bdf45..2038810d5 100644 --- a/lwraft/client/repadmin.c +++ b/lwraft/client/repadmin.c @@ -80,7 +80,7 @@ DWORD VmDirCreateLdAtHostViaMachineAccount( dwError = VmDirStringPrintFA( bufUPN, sizeof(bufUPN)-1, "%s@%s", pszDCAccount, pszDomain); BAIL_ON_VMDIR_ERROR(dwError); - dwError = VmDirSafeLDAPBind( &pLd, pszServerName, bufUPN, pszDCAccountPassword); + dwError = VmDirSafeLDAPBindExt1( &pLd, pszServerName, bufUPN, pszDCAccountPassword, MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); *ppLd = pLd; diff --git a/lwraft/common/ldapbind.c b/lwraft/common/ldapbind.c index 052fc0426..a0e47df06 100644 --- a/lwraft/common/ldapbind.c +++ b/lwraft/common/ldapbind.c @@ -99,6 +99,18 @@ VmDirSASLSRPBind( PCSTR pszUPN, PCSTR pszPass ) +{ + return VmDirSASLSRPBindExt1(ppLd, pszURI, pszUPN, pszPass, -1); // -1 == no timeout +} + +DWORD +VmDirSASLSRPBindExt1( + LDAP** ppLd, + PCSTR pszURI, + PCSTR pszUPN, + PCSTR pszPass, + int iTimeout + ) { DWORD dwError = 0; int retVal = 0; @@ -108,6 +120,10 @@ VmDirSASLSRPBind( const int iSaslNoCanon = 1; VMDIR_SASL_INTERACTIVE_DEFAULT srpDefault = {0}; int iCnt = 0; + struct timeval optTimeout={0}; + + optTimeout.tv_usec = 0; + optTimeout.tv_sec = iTimeout; if ( ppLd == NULL || pszURI == NULL || pszUPN == NULL || pszPass == NULL ) { @@ -133,6 +149,14 @@ VmDirSASLSRPBind( retVal = ldap_set_option(pLd, LDAP_OPT_X_SASL_NOCANON, &iSaslNoCanon); BAIL_ON_SIMPLE_LDAP_ERROR(retVal); + // timeout connect + retVal = ldap_set_option(pLd, LDAP_OPT_TIMEOUT, (void *)&optTimeout); + BAIL_ON_SIMPLE_LDAP_ERROR(retVal); + + // timeout poll + retVal = ldap_set_option(pLd, LDAP_OPT_NETWORK_TIMEOUT, (void *)&optTimeout); + BAIL_ON_SIMPLE_LDAP_ERROR(retVal); + retVal = ldap_sasl_interactive_bind_s( pLd, NULL, "SRP", @@ -141,6 +165,7 @@ VmDirSASLSRPBind( LDAP_SASL_QUIET, _VmDirSASLSRPInteraction, &srpDefault); +#ifndef LIGHTWAVE_BUILD if (retVal == LDAP_SERVER_DOWN) { VmDirSleep(50); // pause 50 ms @@ -152,6 +177,7 @@ VmDirSASLSRPBind( continue; // if transient network error, retry once. } else +#endif { break; } @@ -288,18 +314,29 @@ VmDirSSLBind( goto cleanup; } +DWORD +VmDirSafeLDAPBind( + LDAP** ppLd, + PCSTR pszHost, + PCSTR pszUPN, + PCSTR pszPassword + ) +{ + return VmDirSafeLDAPBindExt1(ppLd, pszHost, pszUPN, pszPassword, -1); // -1 == no timeout +} /* * Bind to partner via "SRP" mechanism. */ DWORD -VmDirSafeLDAPBind( +VmDirSafeLDAPBindExt1( LDAP** ppLd, PCSTR pszHost, PCSTR pszUPN, - PCSTR pszPassword + PCSTR pszPassword, + int iTimeout ) { - return VmDirSafeLDAPBindToPort(ppLd, pszHost, 0, pszUPN, pszPassword); + return VmDirSafeLDAPBindToPort(ppLd, pszHost, 0, pszUPN, pszPassword, iTimeout); } DWORD @@ -308,7 +345,8 @@ VmDirSafeLDAPBindToPort( PCSTR pszHost, DWORD dwPort, PCSTR pszUPN, - PCSTR pszPassword + PCSTR pszPassword, + int iTimeout ) { DWORD dwError = 0; @@ -349,7 +387,7 @@ VmDirSafeLDAPBindToPort( } BAIL_ON_VMDIR_ERROR(dwError); - dwError = VmDirSASLSRPBind( &pLd, &(ldapURI[0]), pszUPN, pszPassword); + dwError = VmDirSASLSRPBindExt1( &pLd, &(ldapURI[0]), pszUPN, pszPassword, iTimeout); BAIL_ON_VMDIR_ERROR(dwError); *ppLd = pLd; @@ -360,8 +398,8 @@ VmDirSafeLDAPBindToPort( error: - VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, "VmDirSafeLDAPBind to (%s) failed. SRP(%d)", - ldapURI, dwError ); + VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, "%s to (%s) failed. SRP(%d)", + __FUNCTION__, ldapURI, dwError ); if ( pLd ) { @@ -384,7 +422,7 @@ VmDirAnonymousLDAPBind( PCSTR pszLdapURI ) { - return VmDirAnonymousLDAPBindWithTimeout(ppLd, pszLdapURI, 0); + return VmDirAnonymousLDAPBindWithTimeout(ppLd, pszLdapURI, MAX_LDAP_CONNECT_NETWORK_TIMEOUT); } @@ -418,6 +456,12 @@ VmDirAnonymousLDAPBindWithTimeout( if (timeout > 0) { nettimeout.tv_sec = timeout; + + // timeout connect + retVal = ldap_set_option( pLocalLd, LDAP_OPT_TIMEOUT, (void *)&nettimeout); + BAIL_ON_SIMPLE_LDAP_ERROR(retVal); + + // timeout poll retVal = ldap_set_option( pLocalLd, LDAP_OPT_NETWORK_TIMEOUT, (void *)&nettimeout); BAIL_ON_SIMPLE_LDAP_ERROR(retVal); } @@ -642,6 +686,8 @@ VmDirMapLdapError( return VMDIR_ERROR_DATA_CONSTRAINT_VIOLATION; case LDAP_BUSY: return VMDIR_ERROR_BUSY; + case LDAP_TIMEOUT: + return VMDIR_ERROR_NETWORK_TIMEOUT; default: return VMDIR_ERROR_GENERIC; } diff --git a/lwraft/config/Makefile.am b/lwraft/config/Makefile.am index bf226f1f2..8755abf8c 100644 --- a/lwraft/config/Makefile.am +++ b/lwraft/config/Makefile.am @@ -6,6 +6,7 @@ lwraftconf_DATA = \ saslpostd.conf \ post-rest.json \ post-telegraf.conf \ - post-demote-deads.sh + post-demote-deads.sh \ + refresh-resolve-conf.sh bin_SCRIPTS = diff --git a/lwraft/config/deployment/aws/appspec.yml b/lwraft/config/deployment/aws/appspec.yml index f528f1079..a30bf217d 100644 --- a/lwraft/config/deployment/aws/appspec.yml +++ b/lwraft/config/deployment/aws/appspec.yml @@ -8,10 +8,6 @@ files: - source: / destination: /var/vmware/lightwave hooks: - BeforeBlockTraffic: - - location: scripts/before_block_traffic.sh - timeout: 300 - runas: root ApplicationStop: - location: scripts/application_stop.sh timeout: 300 diff --git a/lwraft/config/deployment/aws/crontab/post-cron.txt b/lwraft/config/deployment/aws/crontab/post-cron.txt index d683d5f86..92f1d2b76 100644 --- a/lwraft/config/deployment/aws/crontab/post-cron.txt +++ b/lwraft/config/deployment/aws/crontab/post-cron.txt @@ -1,2 +1,2 @@ */3 * * * * /opt/vmware/share/config/post-demote-deads.sh -*/5 * * * * systemctl restart systemd-networkd systemd-resolved +*/5 * * * * /opt/vmware/share/config/refresh-resolve-conf.sh diff --git a/lwraft/config/deployment/aws/scripts/after_allow_traffic.sh b/lwraft/config/deployment/aws/scripts/after_allow_traffic.sh index f8f4ead8c..66820b757 100755 --- a/lwraft/config/deployment/aws/scripts/after_allow_traffic.sh +++ b/lwraft/config/deployment/aws/scripts/after_allow_traffic.sh @@ -6,7 +6,7 @@ source $(dirname $(realpath $0))/common.sh echo "Step 1: Check if localhost is the leader (if yes, continue)" -get_tag_value "POST_PASSWORD" POST_PASSWORD +get_post_password POST_PASSWORD LOCALHOST=`hostname -f | awk '{print tolower($0)}'` LEADER=$(/opt/vmware/bin/post-cli node state --server-name localhost --login administrator --password ${POST_PASSWORD} | grep Leader | awk '{print $1}') diff --git a/lwraft/config/deployment/aws/scripts/after_install.sh b/lwraft/config/deployment/aws/scripts/after_install.sh index 410642d45..84af12d8b 100755 --- a/lwraft/config/deployment/aws/scripts/after_install.sh +++ b/lwraft/config/deployment/aws/scripts/after_install.sh @@ -26,6 +26,7 @@ tdnf makecache tdnf install -y lightwave-post lightwave-client +# TODO - this should not be necessary when DNS is stabilized echo "Step 4: Set proxy curl timeout" /opt/likewise/bin/lwregshell add_value '[HKEY_THIS_MACHINE\Services\post\Parameters]' CurlTimeoutSec REG_DWORD 10 || echo "CurTimeoutSec is already set" diff --git a/lwraft/config/deployment/aws/scripts/application_start.sh b/lwraft/config/deployment/aws/scripts/application_start.sh index 73f7cdbfa..d0e87db4e 100755 --- a/lwraft/config/deployment/aws/scripts/application_start.sh +++ b/lwraft/config/deployment/aws/scripts/application_start.sh @@ -7,12 +7,11 @@ echo "Step 1: Get domain, password, and existing partners from AWS" get_tag_value "LW_DOMAIN" LW_DOMAIN echo "LW_DOMAIN=${LW_DOMAIN}" -get_tag_value "POST_PASSWORD" POST_PASSWORD -echo "POST_PASSWORD=" - find_post_partners PARTNERS echo "PARTNERS=${PARTNERS[*]}" +get_post_password POST_PASSWORD + echo "Step 2: Start POST" diff --git a/lwraft/config/deployment/aws/scripts/before_block_traffic.sh b/lwraft/config/deployment/aws/scripts/before_block_traffic.sh deleted file mode 100755 index f557cf91b..000000000 --- a/lwraft/config/deployment/aws/scripts/before_block_traffic.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -xe - -echo "Step 1: Patch schema (TODO)" -# TODO - - -echo "Step 2: Check schema replication status (TODO)" -# TODO diff --git a/lwraft/config/deployment/aws/scripts/before_install.sh b/lwraft/config/deployment/aws/scripts/before_install.sh index 4b66d056c..5f675a1fe 100755 --- a/lwraft/config/deployment/aws/scripts/before_install.sh +++ b/lwraft/config/deployment/aws/scripts/before_install.sh @@ -3,7 +3,7 @@ echo "Step 1: Upgrade/install createrepo and its dependencies" tdnf makecache -tdnf install -y sed zip unzip createrepo c-rest-engine-1.0.4-2.ph1 +tdnf install -y sed zip unzip createrepo c-rest-engine-1.1-1.ph1 echo "Install patched version of cyrus-sasl" diff --git a/lwraft/config/deployment/aws/scripts/common.sh b/lwraft/config/deployment/aws/scripts/common.sh index 2d42cde4d..3eab999c7 100755 --- a/lwraft/config/deployment/aws/scripts/common.sh +++ b/lwraft/config/deployment/aws/scripts/common.sh @@ -48,6 +48,28 @@ set_tag_value() { aws autoscaling create-or-update-tags --region ${REGION} --tags "ResourceId=${ASG},ResourceType=auto-scaling-group,Key=${TAG},Value=${VALUE},PropagateAtLaunch=true" } +# read a S3 file +read_s3_file() +{ + S3_FILE=$1 + TMP_FILE=/tmp/s3file-${RANDOM} + aws s3 cp ${S3_FILE} ${TMP_FILE} + CONTENT=$(cat ${TMP_FILE}) + rm ${TMP_FILE} + eval "$2=${CONTENT}" +} + +# retrieves post admin password +get_post_password() { + get_tag_value "PASSWORD_PATH" PASSWORD_PATH + if [[ -z ${PASSWORD_PATH} ]] + then + PASSWORD_PATH=s3://cascade-passwords/post-password # default path + fi + read_s3_file ${PASSWORD_PATH} PASSWORD + eval "$1=${PASSWORD}" +} + # lists IP of all nodes in the given autoscaling group get_asg_node_list() { get_current_region REGION diff --git a/lwraft/config/deployment/aws/scripts/validate_service.sh b/lwraft/config/deployment/aws/scripts/validate_service.sh index d1975c4f8..8b87eb923 100755 --- a/lwraft/config/deployment/aws/scripts/validate_service.sh +++ b/lwraft/config/deployment/aws/scripts/validate_service.sh @@ -6,31 +6,33 @@ ERRCNT=0 echo "Step 1: Wait for leader election (mostly for 2nd node promotion)" -get_tag_value "POST_PASSWORD" POST_PASSWORD +get_post_password POST_PASSWORD echo '/opt/vmware/bin/post-cli node state --server-name localhost --login administrator --password ' MAX_RETRY=10 RETRY=1 +LEADER="" -while [ ${RETRY} -le ${MAX_RETRY} ] +while [[ ${RETRY} -le ${MAX_RETRY} ]] do sleep 3 /opt/vmware/bin/post-cli node state --server-name localhost --login administrator --password ${POST_PASSWORD} &> ${LOGDIR}/post_cli_node_state.log RET=$? - echo "Attempt ${RETRY}: ${RET}" - if [ ${RET} -eq 0 ] + LEADER=$(grep 'Leader' ${LOGDIR}/post_cli_node_state.log | awk '{print $1;}') + echo "Attempt ${RETRY}: ${RET} (${LEADER})" + if [[ ${RET} -eq 0 && -n ${LEADER} ]] then break fi let RETRY++ done -if [ ${RET} -ne 0 ] +if [[ ${RET} -ne 0 ]] then echo "Error: Returned ${RET} / Expected 0" let ERRCNT++ -elif [[ -z $(grep 'Leader' ${LOGDIR}/post_cli_node_state.log) ]] +elif [[ -z ${LEADER} ]] then cat ${LOGDIR}/post_cli_node_state.log echo "Error: No POST leader is present" @@ -47,7 +49,7 @@ echo 'ldapsearch -h localhost -p 38900 -x -s base dn' ldapsearch -h localhost -p 38900 -x -s base dn &> ${LOGDIR}/ldapsearch_dseroot.log RET=$? -if [ ${RET} -ne 0 ] +if [[ ${RET} -ne 0 ]] then echo "Error: Returned ${RET} / Expected 0" let ERRCNT++ @@ -67,7 +69,7 @@ echo 'curl -X GET http://localhost:7577/v1/post/ldap?dn=cn%3DDSE%20Root' curl -X GET 'http://localhost:7577/v1/post/ldap?dn=cn%3DDSE%20Root' &> ${LOGDIR}/http_ldapsearch_dseroot.log RET=$? -if [ ${RET} -ne 0 ] +if [[ ${RET} -ne 0 ]] then echo "Error: Returned ${RET} / Expected 0" let ERRCNT++ diff --git a/lwraft/config/post-demote-deads.sh b/lwraft/config/post-demote-deads.sh index bcf1181da..53284c17a 100755 --- a/lwraft/config/post-demote-deads.sh +++ b/lwraft/config/post-demote-deads.sh @@ -10,9 +10,18 @@ logger -t post-demote-deads "Step 1: Read ASG tags" INSTANCE=$(curl -sS http://169.254.169.254/latest/meta-data/instance-id) REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s:\([0-9][0-9]*\)[a-z]*\$:\\1:") ASG=$(aws autoscaling describe-auto-scaling-instances --instance-ids ${INSTANCE} --region ${REGION} --query AutoScalingInstances[].AutoScalingGroupName --output text) - -POST_PASSWORD=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names ${ASG} --region ${REGION} --query AutoScalingGroups[].Tags[?Key==\'POST_PASSWORD\'].Value --output text) LW_DOMAIN=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names ${ASG} --region ${REGION} --query AutoScalingGroups[].Tags[?Key==\'LW_DOMAIN\'].Value --output text) +PASSWORD_PATH=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names ${ASG} --region ${REGION} --query AutoScalingGroups[].Tags[?Key==\'PASSWORD_PATH\'].Value --output text) + +if [[ -z ${PASSWORD_PATH} ]] +then + PASSWORD_PATH=s3://cascade-passwords/post-password # default path +fi + +TMPFILE=/tmp/cronpw-${RANDOM} +aws s3 cp ${PASSWORD_PATH} ${TMPFILE} +POST_PASSWORD=$(cat ${TMPFILE}) +rm ${TMPFILE} logger -t post-demote-deads "Step 2: Check if localhost is the leader (if yes, continue)" diff --git a/lwraft/config/post-rest.json b/lwraft/config/post-rest.json index c6481c98e..56f95a7da 100644 --- a/lwraft/config/post-rest.json +++ b/lwraft/config/post-rest.json @@ -51,7 +51,7 @@ } ], "responses": { - "200": { + "default": { "description": "Generic LDAP response", "schema": { "$ref": "#/definitions/GenericResponse" @@ -116,7 +116,14 @@ "schema": { "$ref": "#/definitions/LDAPSearchResponse" } + }, + "default": { + "description": "Generic LDAP response", + "schema": { + "$ref": "#/definitions/GenericResponse" + } } + }, "tags": [ "ldap" @@ -154,7 +161,7 @@ } ], "responses": { - "200": { + "default": { "description": "Generic LDAP response", "schema": { "$ref": "#/definitions/GenericResponse" @@ -177,7 +184,7 @@ } ], "responses": { - "200": { + "default": { "description": "Generic LDAP response", "schema": { "$ref": "#/definitions/GenericResponse" @@ -219,7 +226,7 @@ } ], "responses": { - "200": { + "default": { "description": "Generic POST object response", "schema": { "$ref": "#/definitions/GenericResponse" @@ -295,6 +302,12 @@ "schema": { "$ref": "#/definitions/PostObjectGetResponse" } + }, + "default": { + "description": "Generic POST object response", + "schema": { + "$ref": "#/definitions/GenericResponse" + } } }, "tags": [ @@ -339,7 +352,7 @@ } ], "responses": { - "200": { + "default": { "description": "Generic POST response", "schema": { "$ref": "#/definitions/GenericResponse" @@ -374,7 +387,7 @@ } ], "responses": { - "200": { + "default": { "description": "Generic POST response", "schema": { "$ref": "#/definitions/GenericResponse" @@ -548,10 +561,6 @@ }, "error_message": { "type": "string" - }, - "result_count": { - "type": "integer", - "format": "int32" } } }, diff --git a/lwraft/config/refresh-resolve-conf.sh b/lwraft/config/refresh-resolve-conf.sh new file mode 100755 index 000000000..fa09f7d63 --- /dev/null +++ b/lwraft/config/refresh-resolve-conf.sh @@ -0,0 +1,35 @@ +#!/bin/bash -e + +export PATH=$PATH:/root/.local/bin + +logger -t refresh-resolve-conf "Starts" + + +logger -t refresh-resolve-conf "Step 1: Get DNS list from DHCP option set" + +INSTANCE=$(curl -sS http://169.254.169.254/latest/meta-data/instance-id) +REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s:\([0-9][0-9]*\)[a-z]*\$:\\1:") +VPC=$(aws ec2 describe-instances --region ${REGION} --instance-ids ${INSTANCE} --query 'Reservations[].Instances[].NetworkInterfaces[].VpcId' --output text) + +DHCP=$(aws ec2 describe-vpcs --region ${REGION} --vpc-ids ${VPC} --query 'Vpcs[].DhcpOptionsId' --output text) +logger -t refresh-resolve-conf "DHCP option set: ${DHCP}" + +DHCP_DNS=($(aws ec2 describe-dhcp-options --region ${REGION} --dhcp-options-ids ${DHCP} --query 'DhcpOptions[].DhcpConfigurations[].Values[].Value' --output text)) +logger -t refresh-resolve-conf "DHCP DNS list: ${DHCP_DNS[@]}" + + +logger -t refresh-resolve-conf "Step 2: Compare the list against /etc/resolv.conf" + +RESOLV_DNS=($(grep nameserver /etc/resolv.conf | awk '{print $2;}')) +logger -t refresh-resolve-conf "/etc/resolv.conf: ${RESOLV_DNS[@]}" + +if [[ ${DHCP_DNS[@]} == ${RESOLV_DNS[@]} ]] +then + logger -t refresh-resolve-conf "/etc/resolv.conf is in sync with DHCP option set, nothing to do" +else + logger -t refresh-resolve-conf "systemctl restart systemd-networkd systemd-resolved" + systemctl restart systemd-networkd systemd-resolved +fi + + +logger -t refresh-resolve-conf "Ends" diff --git a/lwraft/include/public/vmdirerrors.h b/lwraft/include/public/vmdirerrors.h index 394bdb5a7..3a8a79253 100644 --- a/lwraft/include/public/vmdirerrors.h +++ b/lwraft/include/public/vmdirerrors.h @@ -100,7 +100,8 @@ #define VMDIR_ERROR_AUTH_BAD_DATA (VMDIR_ERROR_BASE + VMDIR_GENERIC_ERROR_BASE + 32) // 9132 #define VMDIR_ERROR_AFD_UNAVAILABLE (VMDIR_ERROR_BASE + VMDIR_GENERIC_ERROR_BASE + 33) // 9133 #define VMDIR_ERROR_OIDC_UNAVAILABLE (VMDIR_ERROR_BASE + VMDIR_GENERIC_ERROR_BASE + 34) // 9134 -#define VMDIR_ERROR_ALREADY_PROMOTED (VMDIR_ERROR_BASE + VMDIR_GENERIC_ERROR_BASE + 33) // 9135 +#define VMDIR_ERROR_ALREADY_PROMOTED (VMDIR_ERROR_BASE + VMDIR_GENERIC_ERROR_BASE + 35) // 9135 +#define VMDIR_ERROR_NETWORK_TIMEOUT (VMDIR_ERROR_BASE + VMDIR_GENERIC_ERROR_BASE + 36) // 9136 // SID/ACL 9200 ~9229 #define VMDIR_ERROR_RID_LIMIT_EXCEEDED (VMDIR_ERROR_BASE + VMDIR_GENERIC_ERROR_BASE + 100 ) // 9200 diff --git a/lwraft/include/vmdircommon.h b/lwraft/include/vmdircommon.h index 9d7b5f22d..5f5d76d25 100644 --- a/lwraft/include/vmdircommon.h +++ b/lwraft/include/vmdircommon.h @@ -884,6 +884,7 @@ typedef enum #define VMDIR_REG_KEY_HTTP_LISTEN_PORT "RestListenHTTPPort" #define VMDIR_REG_KEY_HTTPS_LISTEN_PORT "RestListenHTTPSPort" #define VMDIR_REG_KEY_LDAP_RECV_TIMEOUT_SEC "LdapRecvTimeoutSec" +#define VMDIR_REG_KEY_LDAP_CONNECT_TIMEOUT_SEC "LdapConnectTimeoutSec" #define VMDIR_REG_KEY_ALLOW_ADMIN_LOCKOUT "AllowAdminLockout" #define VMDIR_REG_KEY_MAX_OP_THREADS "MaxLdapOpThrs" #define VMDIR_REG_KEY_DISABLE_VECS "DisableVECSIntegration" @@ -1603,6 +1604,15 @@ VmDirSASLSRPBind( PCSTR pszPass ); +DWORD +VmDirSASLSRPBindExt1( + LDAP** ppLd, + PCSTR pszURI, + PCSTR pszUPN, + PCSTR pszPass, + int iTimeout + ); + DWORD VmDirSSLBind( LDAP** ppLd, @@ -1619,13 +1629,23 @@ VmDirSafeLDAPBind( PCSTR pszPassword // opt, if exists, will try SRP mech ); +DWORD +VmDirSafeLDAPBindExt1( + LDAP** ppLd, + PCSTR pszHost, + PCSTR pszUPN, + PCSTR pszPassword, + int iTimeout // connection timeout + ); + DWORD VmDirSafeLDAPBindToPort( LDAP** ppLd, PCSTR pszHost, DWORD dwPort, PCSTR pszUPN, - PCSTR pszPassword + PCSTR pszPassword, + int iTimeout ); DWORD diff --git a/lwraft/include/vmdirdefines.h b/lwraft/include/vmdirdefines.h index 363043328..2027a004f 100644 --- a/lwraft/include/vmdirdefines.h +++ b/lwraft/include/vmdirdefines.h @@ -624,6 +624,8 @@ extern "C" { #define MAX_DEADLOCK_RETRIES 5 +#define MAX_LDAP_CONNECT_NETWORK_TIMEOUT 3 + #ifndef LDAP_DEBUG_ANY #define LDAP_DEBUG_ANY (-1) #endif diff --git a/lwraft/server/include/vmdirserver.h b/lwraft/server/include/vmdirserver.h index dd5e99689..8bec163d4 100644 --- a/lwraft/server/include/vmdirserver.h +++ b/lwraft/server/include/vmdirserver.h @@ -135,9 +135,10 @@ typedef struct _VMDIR_GLOBALS DWORD dwLdapsPort; // Timeout for curl requests DWORD dwProxyCurlTimeout; - PSTR pszHTTPListenPort; - PSTR pszHTTPSListenPort; + DWORD dwHTTPListenPort; + DWORD dwHTTPSListenPort; DWORD dwLdapRecvTimeoutSec; + DWORD dwLdapConnectTimeoutSec; BOOLEAN bIsLDAPPortOpen; // following fields are protected by mutex PVMDIR_MUTEX mutex; @@ -197,6 +198,8 @@ typedef struct _VMDIR_GLOBALS DWORD dwRaftPingIntervalMS; //Raft logs to keep in 1000 DWORD dwRaftKeeplogs; + + SSL_CTX* gpVdirSslCtx; } VMDIR_GLOBALS, *PVMDIR_GLOBALSS; extern VMDIR_GLOBALS gVmdirGlobals; diff --git a/lwraft/server/ldap-head/openssl.c b/lwraft/server/ldap-head/openssl.c index f89c8e9a3..a8d710b79 100644 --- a/lwraft/server/ldap-head/openssl.c +++ b/lwraft/server/ldap-head/openssl.c @@ -135,8 +135,6 @@ Sockbuf_IO opensslBerSockbufIO = { /////////////////////////////////////////////////////////////////////////////// Sockbuf_IO* gpVdirBerSockbufIOOpenssl = &opensslBerSockbufIO; -// initialized in VmDirOpensslInit during startup and used to create SSL* -static SSL_CTX* gpVdirSslCtx = NULL; /////////////////////////////////////////////////////////////////////////////// // ****************** shared static variables end ******************** /////////////////////////////////////////////////////////////////////////////// @@ -347,7 +345,7 @@ _VmDirInitSslCtx( } /* - * Initialize openssl libraries and create a default SSL_CTX - gpVdirSslCtx + * Initialize openssl libraries and create a default SSL_CTX - gVmdirGlobals.gpVdirSslCtx */ DWORD VmDirOpensslInit( @@ -437,7 +435,7 @@ VmDirOpensslInit( BAIL_ON_OPENSSL_ERROR( TRUE, "SSL_CTX_check_private_key"); } - gpVdirSslCtx = pSslCtx; + gVmdirGlobals.gpVdirSslCtx = pSslCtx; gVmdirOpensslGlobals.bSSLInitialized = TRUE; cleanup: @@ -472,9 +470,9 @@ VmDirOpensslShutdown( { DWORD dwSize = 0; - if (gpVdirSslCtx) + if (gVmdirGlobals.gpVdirSslCtx) { - SSL_CTX_free(gpVdirSslCtx); + SSL_CTX_free(gVmdirGlobals.gpVdirSslCtx); } ERR_remove_state(0); @@ -635,7 +633,7 @@ opensslSbSetup( pSbiod->sbiod_sb->sb_fd = *((int *)pArg); - pSsl = SSL_new(gpVdirSslCtx); + pSsl = SSL_new(gVmdirGlobals.gpVdirSslCtx); BAIL_ON_OPENSSL_ERROR( (pSsl == NULL), "SSL_new" ); #ifdef _WIN32 diff --git a/lwraft/server/rest-head/auth.c b/lwraft/server/rest-head/auth.c index eb0fdf69b..65fd70062 100644 --- a/lwraft/server/rest-head/auth.c +++ b/lwraft/server/rest-head/auth.c @@ -112,6 +112,11 @@ VmDirRESTAuthViaBasic( pszPasswd++; dwError = VmDirUPNToDN(pszDecode, &pszBindDN); + // we want this to map to invalid credentials error + if (dwError == VMDIR_ERROR_ENTRY_NOT_FOUND) + { + dwError = VMDIR_ERROR_AUTH_BAD_DATA; + } BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirExternalOperationCreate( diff --git a/lwraft/server/rest-head/defines.h b/lwraft/server/rest-head/defines.h index bdd777ddf..db20b901a 100644 --- a/lwraft/server/rest-head/defines.h +++ b/lwraft/server/rest-head/defines.h @@ -17,14 +17,21 @@ #define VMDIR_HTTP_DEBUGLOGFILE LWRAFT_LOG_DIR VMDIR_PATH_SEPARATOR_STR "post-rest-HTTP.log" #define VMDIR_HTTPS_DEBUGLOGFILE LWRAFT_LOG_DIR VMDIR_PATH_SEPARATOR_STR "post-rest-HTTPS.log" -#define VMDIR_REST_CLIENTCNT "64" -#define VMDIR_REST_WORKERTHCNT "64" +#define VMDIR_REST_CLIENTCNT 64 +#define VMDIR_REST_WORKERTHCNT 64 + +#define VMDIR_REST_CONN_TIMEOUT_SEC 30 +#define VMDIR_MAX_DATA_PER_CONN_MB 25 +#define VMDIR_HTTP_DAEMON_NAME "postd-http"; +#define VMDIR_HTTPS_DAEMON_NAME "postd-https"; +#define VMDIR_REST_STOP_TIMEOUT_SEC 10 #define MAX_REST_PAYLOAD_LENGTH 4096 #define VMDIR_V1_LDAP_RESOURCE "/v1/post/ldap" #define VMDIR_V1_OBJ_RESOURCE "/v1/post/object" #define VMDIR_V1_OBJ_RESOURCE_ALL "/v1/post/object/*" +#define VMDIR_V1_METRICS_RESOURCE "/v1/post/metrics" // Lightwave #define VMDIR_REST_LIGHTWAVE_LDAP_PORT 389 diff --git a/lwraft/server/rest-head/handler.c b/lwraft/server/rest-head/handler.c index 6e1e29cd4..49b8a444e 100644 --- a/lwraft/server/rest-head/handler.c +++ b/lwraft/server/rest-head/handler.c @@ -14,6 +14,12 @@ #include "includes.h" +static +BOOL +_VmDirRESTProcessLocally( + PVDIR_REST_OPERATION pRestOp + ); + /* * We provide this function as callback to c-rest-engine, * c-rest-engine will use this callback upon receiving a request @@ -101,8 +107,13 @@ VmDirRESTRequestHandlerInternal( } else { + dwError = VmDirRESTOperationReadMetadata(pRestOp, pRequest); + BAIL_ON_VMDIR_ERROR(dwError); + VmDirRaftGetRole(&role); - if (role == VDIR_RAFT_ROLE_LEADER) + // if node is leader or the request needs to be processed locally + if (role == VDIR_RAFT_ROLE_LEADER || + _VmDirRESTProcessLocally(pRestOp) ) { dwRestOpErr = VmDirRESTProcessRequest( pRestOp, pRESTHandle, pRequest, paramsCount); @@ -111,6 +122,7 @@ VmDirRESTRequestHandlerInternal( pRestOp, pRESTHandle, ppResponse); BAIL_ON_VMDIR_ERROR(dwError); } + // else if follower proxy to leader else if (role == VDIR_RAFT_ROLE_FOLLOWER) { dwRestOpErr = VmDirRESTForwardRequest( @@ -125,6 +137,12 @@ VmDirRESTRequestHandlerInternal( dwError = VmDirRESTWriteSimpleErrorResponse( pRESTHandle, ppResponse, 503); // 503 = Service Unavailable BAIL_ON_VMDIR_ERROR(dwError); + + VMDIR_LOG_ERROR( + VMDIR_LOG_MASK_ALL, + "Error: %d Node in invalid state no leader. Status returned: 503 to client: %s", + VMDIR_ERROR_NO_LEADER, + VDIR_SAFE_STRING(pRestOp->pszClientIP)); } } @@ -135,10 +153,11 @@ VmDirRESTRequestHandlerInternal( error: VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, - "%s failed, error (%d), rest operation error (%d)", + "%s failed, error (%d), rest operation error (%d) for client: %s", __FUNCTION__, dwError, - dwRestOpErr); + dwRestOpErr, + pRestOp ? VDIR_SAFE_STRING(pRestOp->pszClientIP) : ""); goto cleanup; } @@ -176,6 +195,13 @@ VmDirRESTProcessRequest( dwError = pMethod->pFnImpl((void*)pRestOp, NULL); BAIL_ON_VMDIR_ERROR(dwError); + VMDIR_LOG_INFO( + VMDIR_LOG_MASK_ALL, + "Leader received REST request from: %s request type: %s request URI: %s", + VDIR_SAFE_STRING(pRestOp->pszClientIP), + VDIR_SAFE_STRING(pRestOp->pszMethod), + VDIR_SAFE_STRING(pRestOp->pszPath)); + cleanup: VMDIR_SET_REST_RESULT(pRestOp, NULL, dwError, NULL); return dwError; @@ -183,9 +209,10 @@ VmDirRESTProcessRequest( error: VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, - "%s failed, error (%d)", + "%s failed, error (%d) for client: %s", __FUNCTION__, - dwError); + dwError, + VDIR_SAFE_STRING(pRestOp->pszClientIP)); goto cleanup; } @@ -231,3 +258,31 @@ VmDirRESTWriteSimpleErrorResponse( goto cleanup; } + +static +BOOL +_VmDirRESTProcessLocally( + PVDIR_REST_OPERATION pRestOp + ) +{ + BOOL bReturn = FALSE; + + if (!pRestOp || !pRestOp->pszPath) + { + // This is an invalid case, we will return true so that + // the error is handled locally in the upcoming calls + bReturn = TRUE; + } + else if (VmDirStringStartsWith(pRestOp->pszPath, VMDIR_V1_METRICS_RESOURCE, FALSE)) + { + // currently only metrics API will be an exception + // Any more exceptions should be added here. + bReturn = TRUE; + } + else + { + bReturn = FALSE; + } + + return bReturn; +} diff --git a/lwraft/server/rest-head/ldapapi.c b/lwraft/server/rest-head/ldapapi.c index 24dbfb3de..526c15039 100644 --- a/lwraft/server/rest-head/ldapapi.c +++ b/lwraft/server/rest-head/ldapapi.c @@ -402,17 +402,19 @@ VmDirRESTLdapGetHttpError( case LDAP_TYPE_OR_VALUE_EXISTS: case LDAP_OBJECT_CLASS_VIOLATION: case LDAP_ALREADY_EXISTS: - case LDAP_NO_SUCH_OBJECT: case LDAP_CONSTRAINT_VIOLATION: case LDAP_NOT_ALLOWED_ON_NONLEAF: case LDAP_PROTOCOL_ERROR: httpStatus = HTTP_BAD_REQUEST; break; + case LDAP_NO_SUCH_OBJECT: + httpStatus = HTTP_NOT_FOUND; + break; + case LDAP_INVALID_CREDENTIALS: case LDAP_INSUFFICIENT_ACCESS: case LDAP_AUTH_METHOD_NOT_SUPPORTED: - case LDAP_SASL_BIND_IN_PROGRESS: httpStatus = HTTP_UNAUTHORIZED; break; diff --git a/lwraft/server/rest-head/libmain.c b/lwraft/server/rest-head/libmain.c index c2bc17673..7f0b3a051 100644 --- a/lwraft/server/rest-head/libmain.c +++ b/lwraft/server/rest-head/libmain.c @@ -140,12 +140,12 @@ _VmDirRESTServerInitHTTP( REST_CONF config = {0}; PREST_PROCESSOR pHandlers = &sVmDirHTTPHandlers; PREST_API_MODULE pModule = NULL; + PVMREST_HANDLE pHTTPHandle = NULL; /* - * pszHTTPListenPort can never be NULL because of default values assigned to them - * if Port string is empty, it means user wants to disable corresponding service + * if dwHTTPListenPort is '0' user wants to disable HTTP service */ - if (IsNullOrEmptyString(gVmdirGlobals.pszHTTPListenPort)) + if (gVmdirGlobals.dwHTTPListenPort == 0) { VMDIR_LOG_WARNING( VMDIR_LOG_MASK_ALL, @@ -154,14 +154,23 @@ _VmDirRESTServerInitHTTP( goto cleanup; } - config.pSSLCertificate = RSA_SERVER_CERT; - config.pSSLKey = RSA_SERVER_KEY; - config.pServerPort = gVmdirGlobals.pszHTTPListenPort; - config.pDebugLogFile = VMDIR_HTTP_DEBUGLOGFILE; - config.pClientCount = VMDIR_REST_CLIENTCNT; - config.pMaxWorkerThread = VMDIR_REST_WORKERTHCNT; - - dwError = VmRESTInit(&config, NULL, &gpVdirRestHTTPHandle); + config.serverPort = gVmdirGlobals.dwHTTPListenPort; + config.connTimeoutSec = VMDIR_REST_CONN_TIMEOUT_SEC; + config.maxDataPerConnMB = VMDIR_MAX_DATA_PER_CONN_MB; + config.pSSLContext = NULL; + config.nWorkerThr = VMDIR_REST_WORKERTHCNT; + config.nClientCnt = VMDIR_REST_CLIENTCNT; + config.SSLCtxOptionsFlag = 0; + config.pszSSLCertificate = NULL; + config.pszSSLKey = NULL; + config.pszSSLCipherList = NULL; + config.pszDebugLogFile = VMDIR_HTTP_DEBUGLOGFILE; + config.pszDaemonName = VMDIR_HTTP_DAEMON_NAME; + config.isSecure = FALSE; + config.useSysLog = TRUE; + config.debugLogLevel = VMREST_LOG_LEVEL_INFO; + + dwError = VmRESTInit(&config, &pHTTPHandle); BAIL_ON_VMDIR_ERROR(dwError); for (pModule = gpVdirRestApiDef->pModules; pModule; pModule = pModule->pNext) @@ -170,19 +179,21 @@ _VmDirRESTServerInitHTTP( for (; pEndPoint; pEndPoint = pEndPoint->pNext) { dwError = VmRESTRegisterHandler( - gpVdirRestHTTPHandle, pEndPoint->pszName, pHandlers, NULL); + pHTTPHandle, pEndPoint->pszName, pHandlers, NULL); BAIL_ON_VMDIR_ERROR(dwError); } } - dwError = VmRESTStart(gpVdirRestHTTPHandle); + dwError = VmRESTStart(pHTTPHandle); BAIL_ON_VMDIR_ERROR(dwError); + gpVdirRestHTTPHandle = pHTTPHandle; + cleanup: return dwError; error: - _VmDirRESTServerShutdownHTTP(); + _VmDirFreeRESTHandle(pHTTPHandle); VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, "%s failed with error %d, not going to listen on REST port", @@ -199,43 +210,42 @@ _VmDirRESTServerInitHTTPS( ) { DWORD dwError = 0; - PSTR pszCert = NULL; - PSTR pszKey = NULL; REST_CONF config = {0}; PREST_PROCESSOR pHandlers = &sVmDirHTTPSHandlers; PREST_API_MODULE pModule = NULL; + PVMREST_HANDLE pHTTPSHandle = NULL; /* - * pszHTTPSListenPort can never be NULL because of default values assigned to them - * if Port string is empty, it means user wants to disable corresponding service + * If dwHTTPSListenPort is '0' user wants to disable HTTPS service + * Initializing openssl context is treated as soft fail, gpVdirSslCtx can be NULL + * If gpVdirSslCtx NULL, don't start the service */ - if (IsNullOrEmptyString(gVmdirGlobals.pszHTTPSListenPort)) + if (gVmdirGlobals.dwHTTPSListenPort == 0 || gVmdirGlobals.gpVdirSslCtx == NULL) { VMDIR_LOG_WARNING( VMDIR_LOG_MASK_ALL, - "%s : not listening in HTTP port", + "%s : not listening in HTTPS port", __FUNCTION__); goto cleanup; } - config.pSSLCertificate = NULL; - config.pSSLKey = NULL; - config.pServerPort = gVmdirGlobals.pszHTTPSListenPort; - config.pDebugLogFile = VMDIR_HTTPS_DEBUGLOGFILE; - config.pClientCount = VMDIR_REST_CLIENTCNT; - config.pMaxWorkerThread = VMDIR_REST_WORKERTHCNT; - - dwError = VmRESTInit(&config, NULL, &gpVdirRestHTTPSHandle); - BAIL_ON_VMDIR_ERROR(dwError); - - //Get Certificate and Key from VECS and Set it to Rest Engine - dwError = VmDirGetVecsMachineCert(&pszCert, &pszKey); - BAIL_ON_VMDIR_ERROR(dwError); - - dwError = VmRESTSetSSLInfo(gpVdirRestHTTPSHandle, pszCert, VmDirStringLenA(pszCert)+1, SSL_DATA_TYPE_CERT); - BAIL_ON_VMDIR_ERROR(dwError); - - dwError = VmRESTSetSSLInfo(gpVdirRestHTTPSHandle, pszKey, VmDirStringLenA(pszKey)+1, SSL_DATA_TYPE_KEY); + config.serverPort = gVmdirGlobals.dwHTTPSListenPort; + config.connTimeoutSec = VMDIR_REST_CONN_TIMEOUT_SEC; + config.maxDataPerConnMB = VMDIR_MAX_DATA_PER_CONN_MB; + config.pSSLContext = gVmdirGlobals.gpVdirSslCtx; + config.nWorkerThr = VMDIR_REST_WORKERTHCNT; + config.nClientCnt = VMDIR_REST_CLIENTCNT; + config.SSLCtxOptionsFlag = 0; + config.pszSSLCertificate = NULL; + config.pszSSLKey = NULL; + config.pszSSLCipherList = NULL; + config.pszDebugLogFile = VMDIR_HTTPS_DEBUGLOGFILE; + config.pszDaemonName = VMDIR_HTTPS_DAEMON_NAME; + config.isSecure = TRUE; + config.useSysLog = TRUE; + config.debugLogLevel = VMREST_LOG_LEVEL_INFO; + + dwError = VmRESTInit(&config, &pHTTPSHandle); BAIL_ON_VMDIR_ERROR(dwError); for (pModule = gpVdirRestApiDef->pModules; pModule; pModule = pModule->pNext) @@ -244,21 +254,21 @@ _VmDirRESTServerInitHTTPS( for (; pEndPoint; pEndPoint = pEndPoint->pNext) { dwError = VmRESTRegisterHandler( - gpVdirRestHTTPSHandle, pEndPoint->pszName, pHandlers, NULL); + pHTTPSHandle, pEndPoint->pszName, pHandlers, NULL); BAIL_ON_VMDIR_ERROR(dwError); } } - dwError = VmRESTStart(gpVdirRestHTTPSHandle); + dwError = VmRESTStart(pHTTPSHandle); BAIL_ON_VMDIR_ERROR(dwError); + gpVdirRestHTTPSHandle = pHTTPSHandle; + cleanup: - VMDIR_SAFE_FREE_MEMORY(pszCert); - VMDIR_SAFE_FREE_MEMORY(pszKey); return dwError; error: - _VmDirRESTServerShutdownHTTPS(); + _VmDirFreeRESTHandle(pHTTPSHandle); VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, "%s failed with error %d, not going to listen on REST port (expected before promote)", @@ -294,11 +304,26 @@ _VmDirFreeRESTHandle( PVMREST_HANDLE pHandle ) { + DWORD dwError = 0; PREST_API_MODULE pModule = NULL; if (pHandle) { - VmRESTStop(pHandle); + /* + * REST library have detached threads, maximum time out specified is the max time + * allowed for the threads to finish their execution. + * If finished early, it will return success. + * If not able to finish in specified time, failure will be returned + */ + dwError = VmRESTStop(pHandle, VMDIR_REST_STOP_TIMEOUT_SEC); + if (dwError != 0) + { + VMDIR_LOG_WARNING( + VMDIR_LOG_MASK_ALL, + "%s : Error: %d", + __FUNCTION__, + dwError); + } if (gpVdirRestApiDef) { pModule = gpVdirRestApiDef->pModules; diff --git a/lwraft/server/rest-head/lightwave.c b/lwraft/server/rest-head/lightwave.c index 2df140be5..3b02ce75b 100644 --- a/lwraft/server/rest-head/lightwave.c +++ b/lwraft/server/rest-head/lightwave.c @@ -104,7 +104,7 @@ VmDirRESTGetLightwaveObjectSid( BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirSafeLDAPBindToPort( - &pLd, pszDCName, dwPort, pszAccountUPN, pszPassword); + &pLd, pszDCName, dwPort, pszAccountUPN, pszPassword, gVmdirGlobals.dwLdapConnectTimeoutSec); BAIL_ON_VMDIR_ERROR(dwError); dwError = ldap_search_ext_s( diff --git a/lwraft/server/rest-head/operation.c b/lwraft/server/rest-head/operation.c index 1472f65c4..9ce33148b 100644 --- a/lwraft/server/rest-head/operation.c +++ b/lwraft/server/rest-head/operation.c @@ -67,6 +67,41 @@ VmDirRESTOperationCreate( goto cleanup; } +DWORD +VmDirRESTOperationReadMetadata( + PVDIR_REST_OPERATION pRestOp, + PREST_REQUEST pRequest + ) +{ + DWORD dwError = 0; + PSTR pszTemp = NULL; + + if (!pRestOp || !pRequest) + { + BAIL_WITH_VMDIR_ERROR(dwError, VMDIR_ERROR_INVALID_PARAMETER); + } + + // Get the client IP + dwError = VmRESTGetConnectionInfo(pRequest, &pRestOp->pszClientIP, &pRestOp->dwPort); + BAIL_ON_VMDIR_ERROR(dwError); + + // read request URI. TRUE requests c-rest-engine to decode URI + dwError = VmRESTGetHttpURI(pRequest, TRUE, &pRestOp->pszPath); + BAIL_ON_VMDIR_ERROR(dwError); + + pszTemp = VmDirStringChrA(pRestOp->pszPath, '?'); + if (pszTemp) + { + *pszTemp = '\0'; + } + +cleanup: + return dwError; + +error: + goto cleanup; +} + DWORD VmDirRESTOperationReadRequest( PVDIR_REST_OPERATION pRestOp, @@ -78,7 +113,6 @@ VmDirRESTOperationReadRequest( DWORD dwError = 0; DWORD i = 0, bytesRead = 0; json_error_t jError = {0}; - PSTR pszTmp = NULL; PSTR pszKey = NULL; PSTR pszVal = NULL; PSTR pszInput = NULL; @@ -94,16 +128,6 @@ VmDirRESTOperationReadRequest( dwError = VmRESTGetHttpMethod(pRestReq, &pRestOp->pszMethod); BAIL_ON_VMDIR_ERROR(dwError); - // read request URI - dwError = VmRESTGetHttpURI(pRestReq, &pRestOp->pszPath); - BAIL_ON_VMDIR_ERROR(dwError); - - pszTmp = VmDirStringChrA(pRestOp->pszPath, '?'); - if (pszTmp) - { - *pszTmp = '\0'; - } - // determine resource pRestOp->pResource = VmDirRESTGetResource(pRestOp->pszPath); if (pRestOp->pResource->rscType == VDIR_REST_RSC_UNKNOWN) @@ -324,6 +348,7 @@ VmDirFreeRESTOperation( VMDIR_SAFE_FREE_MEMORY(pRestOp->pszHeaderIfMatch); VMDIR_SAFE_FREE_MEMORY(pRestOp->pszContentType); VMDIR_SAFE_FREE_MEMORY(pRestOp->pszInput); + VMDIR_SAFE_FREE_MEMORY(pRestOp->pszClientIP); if (pRestOp->pjInput) { json_decref(pRestOp->pjInput); diff --git a/lwraft/server/rest-head/prototypes.h b/lwraft/server/rest-head/prototypes.h index 24d9cb497..7c93d0c02 100644 --- a/lwraft/server/rest-head/prototypes.h +++ b/lwraft/server/rest-head/prototypes.h @@ -379,6 +379,12 @@ VmDirRESTOperationCreate( PVDIR_REST_OPERATION* ppRestOp ); +DWORD +VmDirRESTOperationReadMetadata( + PVDIR_REST_OPERATION pRestOp, + PREST_REQUEST pRequest + ); + DWORD VmDirRESTOperationReadRequest( PVDIR_REST_OPERATION pRestOp, diff --git a/lwraft/server/rest-head/proxy.c b/lwraft/server/rest-head/proxy.c index f27077da6..8338edbbb 100644 --- a/lwraft/server/rest-head/proxy.c +++ b/lwraft/server/rest-head/proxy.c @@ -247,8 +247,9 @@ VmDirRESTForwardRequest( VMDIR_LOG_INFO( VMDIR_LOG_MASK_ALL, - "Proxy forwarding done to leader: %s time taken: %d milliseconds", - pszLeader, + "Proxy forwarding done for client: %s to leader: %s time taken: %d ms", + VDIR_SAFE_STRING(pRestOp->pszClientIP), + VDIR_SAFE_STRING(pszLeader), VMDIR_RESPONSE_TIME(VmDirGetTimeInMilliSec()-uiStartTime)); // set error dwCurlError = curl_easy_getinfo(pCurlHandle, CURLINFO_RESPONSE_CODE, &statusCode); @@ -271,12 +272,13 @@ VmDirRESTForwardRequest( error: VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, - "%s Failed with error: %d curl error: %d, time taken %d for leader: %s", + "%s Failed with error: %d curl error: %d, time taken %d ms for client: %s to leader: %s", __FUNCTION__, dwError, dwCurlError, VMDIR_RESPONSE_TIME(VmDirGetTimeInMilliSec()-uiStartTime), - pszLeader ? pszLeader : ""); + VDIR_SAFE_STRING(pRestOp->pszClientIP), + VDIR_SAFE_STRING(pszLeader)); goto cleanup; curlerror: @@ -425,6 +427,10 @@ _VmDirRESTProxyResultGetHttpCode( { *pdwHttpCode = _VmDirRESTCurlToHttpCode(pProxyResult->dwCurlError); } + else if(pProxyResult->dwError == VMDIR_ERROR_NO_LEADER) + { + *pdwHttpCode = 503; + } else { *pdwHttpCode = 500; @@ -643,7 +649,6 @@ _VmDirRESTProxyReadRequest( DWORD len = 0; DWORD i = 0; PSTR pszInput = NULL; - PSTR pszTmp = NULL; PSTR pszKey = NULL; PSTR pszVal = NULL; @@ -656,16 +661,6 @@ _VmDirRESTProxyReadRequest( dwError = VmRESTGetHttpMethod(pRequest, &pRestOp->pszMethod); BAIL_ON_VMDIR_ERROR(dwError); - // request URI - dwError = VmRESTGetHttpURI(pRequest, &pRestOp->pszPath); - BAIL_ON_VMDIR_ERROR(dwError); - - pszTmp = VmDirStringChrA(pRestOp->pszPath, '?'); - if (pszTmp) - { - *pszTmp = '\0'; - } - // auth header dwError = VmRESTGetHttpHeader(pRequest, VMDIR_REST_HEADER_AUTHENTICATION, &pRestOp->pszAuth); BAIL_ON_VMDIR_ERROR(dwError); diff --git a/lwraft/server/rest-head/structs.h b/lwraft/server/rest-head/structs.h index a6bf5cf33..78c0462d8 100644 --- a/lwraft/server/rest-head/structs.h +++ b/lwraft/server/rest-head/structs.h @@ -80,6 +80,7 @@ typedef struct _VDIR_PROXY_RESULT typedef struct _VDIR_REST_OPERATION { + DWORD dwPort; PSTR pszAuth; PSTR pszMethod; PSTR pszPath; @@ -87,6 +88,7 @@ typedef struct _VDIR_REST_OPERATION PSTR pszHeaderIfMatch; PSTR pszContentType; PSTR pszInput; + PSTR pszClientIP; json_t* pjInput; PLW_HASHMAP pParamMap; VDIR_REST_AUTH_METHOD authMthd; diff --git a/lwraft/server/vmdir/defines.h b/lwraft/server/vmdir/defines.h index a84496edc..c4f1111cd 100644 --- a/lwraft/server/vmdir/defines.h +++ b/lwraft/server/vmdir/defines.h @@ -178,24 +178,24 @@ }, \ { \ /*.pszName = */ VMDIR_REG_KEY_HTTP_LISTEN_PORT, \ - /*.Type = */ VMDIR_CONFIG_VALUE_TYPE_STRING, \ - /*.RegDataType = */ REG_SZ, \ + /*.Type = */ VMDIR_CONFIG_VALUE_TYPE_DWORD, \ + /*.RegDataType = */ REG_DWORD, \ /*.dwMin = */ 0, \ - /*.dwMax = */ 0, \ - /*.dwDefault = */ 0, \ + /*.dwMax = */ 65535, \ + /*.dwDefault = */ DEFAULT_HTTP_PORT_NUM, \ /*.dwValue = */ 0, \ - /*.pszDefault = */ DEFAULT_HTTP_PORT_STR, \ + /*.pszDefault = */ NULL, \ /*.pszValue = */ NULL \ }, \ { \ /*.pszName = */ VMDIR_REG_KEY_HTTPS_LISTEN_PORT, \ - /*.Type = */ VMDIR_CONFIG_VALUE_TYPE_STRING, \ - /*.RegDataType = */ REG_SZ, \ + /*.Type = */ VMDIR_CONFIG_VALUE_TYPE_DWORD, \ + /*.RegDataType = */ REG_DWORD, \ /*.dwMin = */ 0, \ - /*.dwMax = */ 0, \ - /*.dwDefault = */ 0, \ + /*.dwMax = */ 65535, \ + /*.dwDefault = */ DEFAULT_HTTPS_PORT_NUM, \ /*.dwValue = */ 0, \ - /*.pszDefault = */ DEFAULT_HTTPS_PORT_STR, \ + /*.pszDefault = */ NULL, \ /*.pszValue = */ NULL \ }, \ { \ @@ -209,6 +209,17 @@ /*.pszDefault = */ NULL, \ /*.pszValue = */ NULL \ }, \ + { \ + /*.pszName = */ VMDIR_REG_KEY_LDAP_CONNECT_TIMEOUT_SEC, \ + /*.Type = */ VMDIR_CONFIG_VALUE_TYPE_DWORD, \ + /*.RegDataType = */ REG_DWORD, \ + /*.dwMin = */ 0, \ + /*.dwMax = */ -1, \ + /*.dwDefault = */ 3, \ + /*.dwValue = */ 0, \ + /*.pszDefault = */ NULL, \ + /*.pszValue = */ NULL \ + }, \ { \ /*.pszName = */ VMDIR_REG_KEY_MAX_OP_THREADS, \ /*.Type = */ VMDIR_CONFIG_VALUE_TYPE_DWORD, \ @@ -324,7 +335,7 @@ /*.RegDataType = */ REG_DWORD, \ /*.dwMin = */ 200, \ /*.dwMax = */ 90000, \ - /*.dwDefault = */ 10000, \ + /*.dwDefault = */ 3500, \ /*.dwValue = */ 0, \ /*.pszDefault = */ NULL, \ /*.pszValue = */ NULL \ @@ -335,7 +346,7 @@ /*.RegDataType = */ REG_DWORD, \ /*.dwMin = */ 100, \ /*.dwMax = */ 30000, \ - /*.dwDefault = */ 3000, \ + /*.dwDefault = */ 1000, \ /*.dwValue = */ 0, \ /*.pszDefault = */ NULL, \ /*.pszValue = */ NULL \ diff --git a/lwraft/server/vmdir/globals.c b/lwraft/server/vmdir/globals.c index ff5987de5..3ded243f2 100644 --- a/lwraft/server/vmdir/globals.c +++ b/lwraft/server/vmdir/globals.c @@ -44,9 +44,10 @@ VMDIR_GLOBALS gVmdirGlobals = VMDIR_SF_INIT(.dwLdapPort, DEFAULT_LDAP_PORT_NUM), VMDIR_SF_INIT(.dwLdapsPort, DEFAULT_LDAPS_PORT_NUM), VMDIR_SF_INIT(.dwProxyCurlTimeout, 0), - VMDIR_SF_INIT(.pszHTTPListenPort, NULL), - VMDIR_SF_INIT(.pszHTTPSListenPort, NULL), + VMDIR_SF_INIT(.dwHTTPListenPort, 0), + VMDIR_SF_INIT(.dwHTTPSListenPort, 0), VMDIR_SF_INIT(.dwLdapRecvTimeoutSec, 0), + VMDIR_SF_INIT(.dwLdapConnectTimeoutSec, 0), VMDIR_SF_INIT(.bIsLDAPPortOpen, FALSE), VMDIR_SF_INIT(.mutex, NULL), VMDIR_SF_INIT(.vmdirdState, VMDIRD_STATE_UNDEFINED), @@ -85,6 +86,7 @@ VMDIR_GLOBALS gVmdirGlobals = VMDIR_SF_INIT(.dwEnableRaftReferral, 0), VMDIR_SF_INIT(.dwRaftElectionTimeoutMS, 2100), VMDIR_SF_INIT(.dwRaftPingIntervalMS, 1000), + VMDIR_SF_INIT(.gpVdirSslCtx, NULL), }; VMDIR_KRB_GLOBALS gVmdirKrbGlobals = diff --git a/lwraft/server/vmdir/regconfig.c b/lwraft/server/vmdir/regconfig.c index 00b295b72..4fe314df9 100644 --- a/lwraft/server/vmdir/regconfig.c +++ b/lwraft/server/vmdir/regconfig.c @@ -131,20 +131,14 @@ VmDirSrvUpdateConfig( VMDIR_REG_KEY_HTTP_LISTEN_PORT, TRUE)) { - dwError = VmDirAllocateStringA( - pEntry->pszValue, - &gVmdirGlobals.pszHTTPListenPort); - BAIL_ON_VMDIR_ERROR(dwError); + gVmdirGlobals.dwHTTPListenPort = pEntry->dwValue; } else if (!VmDirStringCompareA( pEntry->pszName, VMDIR_REG_KEY_HTTPS_LISTEN_PORT, TRUE)) { - dwError = VmDirAllocateStringA( - pEntry->pszValue, - &gVmdirGlobals.pszHTTPSListenPort); - BAIL_ON_VMDIR_ERROR(dwError); + gVmdirGlobals.dwHTTPSListenPort = pEntry->dwValue; } else if (!VmDirStringCompareA( pEntry->pszName, @@ -251,6 +245,13 @@ VmDirSrvUpdateConfig( { gVmdirGlobals.dwProxyCurlTimeout = pEntry->dwValue; } + else if (!VmDirStringCompareA( + pEntry->pszName, + VMDIR_REG_KEY_LDAP_CONNECT_TIMEOUT_SEC, + TRUE)) + { + gVmdirGlobals.dwLdapConnectTimeoutSec = pEntry->dwValue; + } } cleanup: diff --git a/lwraft/server/vmdir/shutdown.c b/lwraft/server/vmdir/shutdown.c index 05ef8be77..d66004931 100644 --- a/lwraft/server/vmdir/shutdown.c +++ b/lwraft/server/vmdir/shutdown.c @@ -201,8 +201,6 @@ VmDirCleanupGlobals( // Free vmdir global 'gVmdirGlobals' upon shutdown VMDIR_SAFE_FREE_MEMORY(gVmdirGlobals.pszBDBHome); VMDIR_SAFE_FREE_MEMORY(gVmdirGlobals.pszBootStrapSchemaFile); - VMDIR_SAFE_FREE_MEMORY(gVmdirGlobals.pszHTTPListenPort); - VMDIR_SAFE_FREE_MEMORY(gVmdirGlobals.pszHTTPSListenPort); VMDIR_SAFE_FREE_MUTEX( gVmdirGlobals.replCycleDoneMutex ); VMDIR_SAFE_FREE_MUTEX( gVmdirGlobals.replAgrsMutex ); diff --git a/lwraft/thirdparty/openldap/libraries/mdb/lmdb.h b/lwraft/thirdparty/openldap/libraries/mdb/lmdb.h index 532c3f165..565350bac 100644 --- a/lwraft/thirdparty/openldap/libraries/mdb/lmdb.h +++ b/lwraft/thirdparty/openldap/libraries/mdb/lmdb.h @@ -1534,6 +1534,8 @@ int mdb_reader_check(MDB_env *env, int *dead); int mdb_env_set_state(MDB_env *env, MDB_state_op op, unsigned long *last_xlog_num, unsigned long *dbSizeMb, unsigned long *dbMapSizeMb, char *db_path, int db_path_size); +unsigned long long mdb_env_get_lasttid(MDB_env *env); + /** @} */ #ifdef __cplusplus diff --git a/lwraft/thirdparty/openldap/libraries/mdb/mdb.c b/lwraft/thirdparty/openldap/libraries/mdb/mdb.c index 549762f9c..eb60f9a70 100644 --- a/lwraft/thirdparty/openldap/libraries/mdb/mdb.c +++ b/lwraft/thirdparty/openldap/libraries/mdb/mdb.c @@ -8772,6 +8772,7 @@ int commit_xlog_txn(MDB_env *env, MDB_ID2L xlog_pgs, int start, int end) char *s_pos = xlog_pgs[i+j].mptr; pwrite(env->me_fd, s_pos, env->me_psize, d_offset); } + mdb_eassert(env, p->mp_pages > 0); i += p->mp_pages; } else if (F_ISSET(p->mp_flags, P_META)) { @@ -8853,7 +8854,7 @@ int mdb_rollforward_file(MDB_env *env, char * xlog_file) rc = ENOMEM; goto cleanup; } - p = malloc(env->me_psize); + p = aligned_alloc(env->me_psize, env->me_psize); if (p == NULL) { rc = ENOMEM; @@ -8949,6 +8950,11 @@ int mdb_rollxlogs(MDB_env *env, int purge) #endif MDB_IDL xlog_ids = mdb_midl_alloc(MDB_IDL_UM_MAX); + if(!xlog_ids) + { + rc = ENOMEM; + goto done; + } #ifdef _WIN32 sprintf(xlog_file_dir, "%s\\xlogs\\1*", env->me_path); @@ -8967,12 +8973,7 @@ int mdb_rollxlogs(MDB_env *env, int purge) xlog_num=atol(ffd.cFileName); if(xlog_num>=XLOG_MIN_NUM && xlog_num <= XLOG_MAX_NUM) { - rc = mdb_midl_xappend(xlog_ids, xlog_num); - if (rc) - { - rc = ENOMEM; - goto done; - } + mdb_midl_xappend(xlog_ids, xlog_num); } } while (1) @@ -9033,7 +9034,7 @@ int mdb_rollxlogs(MDB_env *env, int purge) mdb_eassert(env, rc == 0); goto done; } - DPRINTF(("MDB recover is needed; roll forward %ld transaction log files...", c)); + DPRINTF(("MDB recover is needed; roll forward %ld transaction log files...", i)); } for (; i; i--) @@ -9174,6 +9175,7 @@ int wal_sync_meta(MDB_env *env, txnid_t tid) if (env->me_walstate.xlog_pages >= MAX_WAL_PGS) { + mdb_eassert(env, env->me_walstate.xlog_fd != INVALID_HANDLE_VALUE); close(env->me_walstate.xlog_fd); env->me_walstate.xlog_fd = INVALID_HANDLE_VALUE; env->me_walstate.xlog_num++; @@ -9437,4 +9439,13 @@ int write_wal_pages(MDB_env *env, const struct iovec *iov, int n) } #endif +unsigned long long +mdb_env_get_lasttid(MDB_env *env) +{ + if (env->me_metas[1]->mm_txnid > env->me_metas[0]->mm_txnid) + { + return env->me_metas[1]->mm_txnid; + } + return env->me_metas[0]->mm_txnid; +} /** @} */ diff --git a/lwraft/tools/vdcaclmgr/main.c b/lwraft/tools/vdcaclmgr/main.c index 50bda3a6f..fd93481f1 100644 --- a/lwraft/tools/vdcaclmgr/main.c +++ b/lwraft/tools/vdcaclmgr/main.c @@ -219,11 +219,12 @@ VmDirMain(int argc, char* argv[]) dwError = VdcGetUsersPassword(&State, pszPasswordBuf, VMDIR_ARRAY_SIZE(pszPasswordBuf)); BAIL_ON_VMDIR_ERROR(dwError); - dwError = VmDirSafeLDAPBind( + dwError = VmDirSafeLDAPBindExt1( &pLd, State.pszServerName, State.pszUserName, - pszPasswordBuf); + pszPasswordBuf, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); dwError = VdcLoadUsersAndGroups(pLd, State.pszBaseDN, &pUserToSidMapping, &pSidToUserMapping); diff --git a/lwraft/tools/vdcadmintool/ldapbindclient.c b/lwraft/tools/vdcadmintool/ldapbindclient.c index 50d6ae700..01dceedf8 100644 --- a/lwraft/tools/vdcadmintool/ldapbindclient.c +++ b/lwraft/tools/vdcadmintool/ldapbindclient.c @@ -253,7 +253,7 @@ _VdcadminClientTestSRPBind( DWORD dwError = 0; LDAP * pLD = NULL; - dwError = VmDirSASLSRPBind( &pLD, pszLDAPURI, pszDefaultBindUPN, pszDefaultPasswd ); + dwError = VmDirSASLSRPBindExt1( &pLD, pszLDAPURI, pszDefaultBindUPN, pszDefaultPasswd, MAX_LDAP_CONNECT_NETWORK_TIMEOUT ); BAIL_ON_VMDIR_ERROR(dwError); printf("%s SRP bind succeeded.\n\n", pszLDAPURI); diff --git a/lwraft/tools/vdcschema/conn.c b/lwraft/tools/vdcschema/conn.c index 838732cc0..38097d741 100644 --- a/lwraft/tools/vdcschema/conn.c +++ b/lwraft/tools/vdcschema/conn.c @@ -85,6 +85,7 @@ VdcSchemaConnOpen( ) { DWORD dwError = 0; + PSTR pszLeader = NULL; if (!pConn) { @@ -101,14 +102,29 @@ VdcSchemaConnOpen( BAIL_ON_VMDIR_ERROR(dwError); } - dwError = VmDirSafeLDAPBind(&pConn->pLd, - pConn->pszHostName, - pConn->pszUPN, - pConn->pszPassword); + // Get the leader for this hostname + dwError = VmDirRaftLeader(pConn->pszHostName, &pszLeader); BAIL_ON_VMDIR_ERROR(dwError); + if (!pszLeader) + { + BAIL_WITH_VMDIR_ERROR(dwError, VMDIR_ERROR_NO_LEADER); + } -error: + // always connect to leader + dwError = VmDirSafeLDAPBindExt1( + &pConn->pLd, + pszLeader, + pConn->pszUPN, + pConn->pszPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); + BAIL_ON_VMDIR_ERROR(dwError); + +cleanup: + VMDIR_SAFE_FREE_MEMORY(pszLeader); return dwError; + +error: + goto cleanup; } VOID diff --git a/support/developer/scripts/docker-create-network b/support/developer/scripts/docker-create-network new file mode 100755 index 000000000..6616533cb --- /dev/null +++ b/support/developer/scripts/docker-create-network @@ -0,0 +1,7 @@ +#!/bin/bash + +docker network create \ + --subnet=192.168.114.0/27 \ + -o com.docker.network.bridge.enable_ip_masquerade=false \ + lightwave + diff --git a/support/developer/scripts/make-lw-container-0 b/support/developer/scripts/make-lw-container-0 new file mode 100755 index 000000000..f84677b27 --- /dev/null +++ b/support/developer/scripts/make-lw-container-0 @@ -0,0 +1,66 @@ +#!/bin/bash + +function check_lightwave +{ + node=$1 + # Check if lightwave server is up + attempts=1 + reachable="false" + total_attempts=50 + while [ $attempts -lt $total_attempts ] && [ $reachable != "true" ]; do + http_code=$(curl -I -so /dev/null -w "%{response_code}" -s -X GET --insecure https://$node) || true + # The curl returns 000 when it fails to connect to the lightwave server + if [ "$http_code" == "000" ]; then + echo "Lightwave REST server $node not reachable (attempt $attempts/$total_attempts), will try again." + attempts=$[$attempts+1] + sleep 5 + else + reachable="true" + break + fi + done + + if [ $attempts -eq $total_attempts ]; then + echo "Could not connect to Lightwave REST client at $node after $total_attempts attempts" + exit 1 + else + echo "Lightwave server at $node has been successfully deployed." + fi +} + +LIGHTWAVE_DOMAIN=lightwave.local +LIGHTWAVE_PASSWORD='Admin!23' +LIGHTWAVE_SITE=Default-first-site +LIGHTWAVE_HOSTNAME=lw-0.lightwave.local + +LIGHTWAVE_HOME=$HOME/lightwave +LIGHTWAVE_CONFIG_DIR=$LIGHTWAVE_HOME/config-lw-0 +LIGHTWAVE_CONFIG_PATH=$LIGHTWAVE_CONFIG_DIR/lightwave-server.cfg + +mkdir -p $LIGHTWAVE_CONFIG_DIR + +cat << EOF > $LIGHTWAVE_CONFIG_PATH +deployment=standalone +domain=$LIGHTWAVE_DOMAIN +hostname=$LIGHTWAVE_HOSTNAME +admin=Administrator +password=$LIGHTWAVE_PASSWORD +site-name=$LIGHTWAVE_SITE +first-instance=true +EOF + +docker run -d \ + --name lightwave-lw-0 \ + --privileged \ + --net lightwave \ + --hostname $LIGHTWAVE_HOSTNAME \ + --ip 192.168.114.3 \ + --dns 192.168.114.3 \ + --dns 192.168.114.4 \ + --dns 192.168.114.5 \ + --dns-search lightwave.local \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v $LIGHTWAVE_CONFIG_DIR:/var/lib/vmware/config \ + vmware/lightwave-sts + +check_lightwave 192.168.114.3 diff --git a/support/developer/scripts/make-lw-container-1 b/support/developer/scripts/make-lw-container-1 new file mode 100755 index 000000000..5011459c1 --- /dev/null +++ b/support/developer/scripts/make-lw-container-1 @@ -0,0 +1,68 @@ +#!/bin/bash + +function check_lightwave +{ + node=$1 + # Check if lightwave server is up + attempts=1 + reachable="false" + total_attempts=50 + while [ $attempts -lt $total_attempts ] && [ $reachable != "true" ]; do + http_code=$(curl -I -so /dev/null -w "%{response_code}" -s -X GET --insecure https://$node) || true + # The curl returns 000 when it fails to connect to the lightwave server + if [ "$http_code" == "000" ]; then + echo "Lightwave REST server $node not reachable (attempt $attempts/$total_attempts), will try again." + attempts=$[$attempts+1] + sleep 5 + else + reachable="true" + break + fi + done + + if [ $attempts -eq $total_attempts ]; then + echo "Could not connect to Lightwave REST client at $node after $total_attempts attempts" + exit 1 + else + echo "Lightwave server at $node has been successfully deployed." + fi +} + +LIGHTWAVE_DOMAIN=lightwave.local +LIGHTWAVE_PASSWORD='Admin!23' +LIGHTWAVE_SITE=Default-first-site +LIGHTWAVE_HOSTNAME=lw-1.lightwave.local +LIGHTWAVE_PARTNER_IP=$(docker inspect lightwave-lw-0 --format '{{ .NetworkSettings.Networks.lightwave.IPAddress}}') + +LIGHTWAVE_HOME=$HOME/lightwave +LIGHTWAVE_CONFIG_DIR=$LIGHTWAVE_HOME/config-lw-1 +LIGHTWAVE_CONFIG_PATH=$LIGHTWAVE_CONFIG_DIR/lightwave-server.cfg + +mkdir -p $LIGHTWAVE_CONFIG_DIR + +cat << EOF > $LIGHTWAVE_CONFIG_PATH +deployment=partner +domain=$LIGHTWAVE_DOMAIN +hostname=$LIGHTWAVE_HOSTNAME +admin=Administrator +password=$LIGHTWAVE_PASSWORD +site-name=$LIGHTWAVE_SITE +first-instance=false +replication-partner-hostname=$LIGHTWAVE_PARTNER_IP +EOF + +docker run -d \ + --name lightwave-lw-1 \ + --privileged \ + --net lightwave \ + --hostname $LIGHTWAVE_HOSTNAME \ + --ip 192.168.114.4 \ + --dns 192.168.114.3 \ + --dns 192.168.114.4 \ + --dns 192.168.114.5 \ + --dns-search lightwave.local \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v $LIGHTWAVE_CONFIG_DIR:/var/lib/vmware/config \ + vmware/lightwave-sts + +check_lightwave 192.168.114.4 diff --git a/support/developer/scripts/make-lw-container-2 b/support/developer/scripts/make-lw-container-2 new file mode 100755 index 000000000..13a77b6c0 --- /dev/null +++ b/support/developer/scripts/make-lw-container-2 @@ -0,0 +1,69 @@ +#!/bin/bash + +function check_lightwave +{ + node=$1 + # Check if lightwave server is up + attempts=1 + reachable="false" + total_attempts=50 + while [ $attempts -lt $total_attempts ] && [ $reachable != "true" ]; do + http_code=$(curl -I -so /dev/null -w "%{response_code}" -s -X GET --insecure https://$node) || true + # The curl returns 000 when it fails to connect to the lightwave server + if [ "$http_code" == "000" ]; then + echo "Lightwave REST server $node not reachable (attempt $attempts/$total_attempts), will try again." + attempts=$[$attempts+1] + sleep 5 + else + reachable="true" + break + fi + done + + if [ $attempts -eq $total_attempts ]; then + echo "Could not connect to Lightwave REST client at $node after $total_attempts attempts" + exit 1 + else + echo "Lightwave server at $node has been successfully deployed." + fi +} + +LIGHTWAVE_DOMAIN=lightwave.local +LIGHTWAVE_PASSWORD='Admin!23' +LIGHTWAVE_SITE=Default-first-site +LIGHTWAVE_HOSTNAME=lw-2.lightwave.local +LIGHTWAVE_PARTNER_IP=$(docker inspect lightwave-lw-0 --format '{{ .NetworkSettings.Networks.lightwave.IPAddress}}') + +LIGHTWAVE_HOME=$HOME/lightwave +LIGHTWAVE_CONFIG_DIR=$LIGHTWAVE_HOME/config-lw-2 +LIGHTWAVE_CONFIG_PATH=$LIGHTWAVE_CONFIG_DIR/lightwave-server.cfg + +mkdir -p $LIGHTWAVE_CONFIG_DIR + +cat << EOF > $LIGHTWAVE_CONFIG_PATH +deployment=partner +domain=$LIGHTWAVE_DOMAIN +hostname=$LIGHTWAVE_HOSTNAME +admin=Administrator +password=$LIGHTWAVE_PASSWORD +site-name=$LIGHTWAVE_SITE +first-instance=false +replication-partner-hostname=$LIGHTWAVE_PARTNER_IP +EOF + +docker run -d \ + --name lightwave-lw-2 \ + --privileged \ + --net lightwave \ + --hostname $LIGHTWAVE_HOSTNAME \ + --ip 192.168.114.5 \ + --dns 192.168.114.3 \ + --dns 192.168.114.4 \ + --dns 192.168.114.5 \ + --dns-search lightwave.local \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v $LIGHTWAVE_CONFIG_DIR:/var/lib/vmware/config \ + vmware/lightwave-sts + +check_lightwave 192.168.114.5 + diff --git a/support/docker/Dockerfile.client b/support/docker/Dockerfile.client deleted file mode 100644 index e0700f6ab..000000000 --- a/support/docker/Dockerfile.client +++ /dev/null @@ -1,37 +0,0 @@ -FROM vmware/photon:latest -MAINTAINER "Jonathan Brown" -ENV container=docker -VOLUME ["/sys/fs/cgroup"] - -# install systemd -RUN tdnf update --refresh -y tdnf; \ - tdnf update -y rpm; \ - tdnf install -y systemd; \ - # Remove unused systemd services - rm -f /etc/systemd/system/*.wants/*;\ - rm -f /lib/systemd/system/sysinit.target.wants/systemd-tmpfiles-setup.service;\ - rm -f /lib/systemd/system/multi-user.target.wants/*;\ - rm -f /lib/systemd/system/local-fs.target.wants/*; \ - rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ - rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ - mkdir -p /var/run/sshd; chmod -rx /var/run/sshd; \ - # configure journald - tdnf install -y sed; \ - sed -i 's/#Storage=auto/Storage=persistent/' /etc/systemd/journald.conf; \ - tdnf install -y procps-ng; \ - tdnf install -y commons-daemon \ - apache-tomcat \ - boost; \ - tdnf install -y likewise-open-6.2.10; \ - tdnf install -y vmware-lightwave-clients; \ - rm -rf /usr/share/doc/*; \ - rm -rf /usr/share/man/*; \ - rm -rf /usr/include/*; - -ADD configure-lightwave-client.service /usr/lib/systemd/system/ -RUN ln -s /usr/lib/systemd/system/configure-lightwave-client.service \ - /etc/systemd/system/multi-user.target.wants/configure-lightwave-client.service - -EXPOSE 22 - -ENTRYPOINT ["/usr/sbin/init"] diff --git a/support/docker/base/Dockerfile b/support/docker/base/Dockerfile new file mode 100644 index 000000000..8ebe99024 --- /dev/null +++ b/support/docker/base/Dockerfile @@ -0,0 +1,24 @@ +FROM vmware/photon:1.0 +MAINTAINER "Jonathan Brown" +ENV container=docker +VOLUME ["/sys/fs/cgroup"] +LABEL vendor="VMware, Inc." +LABEL com.vmware.lightwave-base.version="1.0.0" + +RUN tdnf update --refresh -y \ + tdnf-1.1.0 \ + rpm-4.13.0.1 && \ + tdnf install -y \ + openssl-1.0.2h \ + createrepo-0.10.4 \ + apache-tomcat-8.5.20 \ + boost-1.60.0 \ + commons-daemon-1.0.15 \ + likewise-open-6.2.11 \ + openjre-1.8.0.141 \ + procps-ng-3.3.11 \ + sed-4.2.2 \ + jansson-2.9 \ + gawk-4.1.3 \ + copenapi-0.0.1 \ + c-rest-engine-1.1 diff --git a/support/docker/build-lightwave-client-container.sh b/support/docker/build-lightwave-client-container.sh deleted file mode 100755 index 953c520ac..000000000 --- a/support/docker/build-lightwave-client-container.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -if [ $# -lt 2 ]; then - echo "Usage: build-lightwave-client-container.sh " - exit 1 -fi - -DOCKERFILE_PATH=$1 -DOCKER_IMAGE=$2 - -DOCKER_IMAGE_TAG="vmware/lightwave-client" - -# start docker daemon -systemctl start docker -if [ $? -ne 0 ]; then - echo "Failed to start Docker service" - exit 1 -fi - -sleep 5 - -# create lightwave yum repository - -createrepo $DOCKERFILE_PATH - -# modify Dockerfile to use local lightwave yum repository - -tmpfile=$(mktemp /tmp/lw.XXXXXX) -cat >$tmpfile < $DOCKER_IMAGE -if [ $? -ne 0 ]; then - echo "Failed to export docker image" - exit 1 -fi - -echo "Build docker image successfully at [$DOCKER_IMAGE]" diff --git a/support/docker/build-lightwave-container.sh b/support/docker/build-lightwave-container.sh deleted file mode 100755 index 15a3c7186..000000000 --- a/support/docker/build-lightwave-container.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -if [ $# -lt 2 ]; then - echo "Usage: build-lightwave-container.sh " - exit 1 -fi - -DOCKERFILE_PATH=$1 -DOCKER_IMAGE=$2 - -DOCKER_IMAGE_TAG="vmware/lightwave-sts" - -# start docker daemon -systemctl start docker -if [ $? -ne 0 ]; then - echo "Failed to start Docker service" - exit 1 -fi - -sleep 5 - -# create lightwave yum repository - -createrepo $DOCKERFILE_PATH - -# modify Dockerfile to use local lightwave yum repository - -tmpfile=$(mktemp /tmp/lw.XXXXXX) -cat >$tmpfile < $DOCKER_IMAGE -if [ $? -ne 0 ]; then - echo "Failed to export docker image" - exit 1 -fi - -echo "Build docker image successfully at [$DOCKER_IMAGE]" diff --git a/support/docker/configure-identity-server.service b/support/docker/configure-identity-server.service deleted file mode 100644 index 808408e4b..000000000 --- a/support/docker/configure-identity-server.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Identity Server Configuration Service -After=syslog.target network.target lwsmd.service -Requires=lwsmd.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/opt/vmware/bin/configure-identity-server -KillMode=none - -[Install] -WantedBy=multi-user.target diff --git a/support/docker/configure-lightwave-client.service b/support/docker/configure-lightwave-client.service deleted file mode 100644 index 8e66dc00e..000000000 --- a/support/docker/configure-lightwave-client.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Lightwave Client Configuration Service -After=syslog.target network.target lwsmd.service -Requires=lwsmd.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/opt/vmware/bin/ic-join --help - -[Install] -WantedBy=multi-user.target diff --git a/support/docker/configure-lightwave-server.service b/support/docker/configure-lightwave-server.service deleted file mode 100644 index c028eb169..000000000 --- a/support/docker/configure-lightwave-server.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Lightwave Server Configuration Service -After=syslog.target network.target -Before=lwsmd.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/opt/vmware/bin/configure-lightwave-server --config-file /var/lib/vmware/config/lightwave-server.cfg -KillMode=none - -[Install] -WantedBy=multi-user.target diff --git a/support/docker/Dockerfile b/support/docker/sts/Dockerfile similarity index 57% rename from support/docker/Dockerfile rename to support/docker/sts/Dockerfile index 9324ed876..5555f0524 100644 --- a/support/docker/Dockerfile +++ b/support/docker/sts/Dockerfile @@ -1,39 +1,16 @@ -FROM vmware/photon:1.0 -MAINTAINER "Andrew Gormley" +FROM vmware/lightwave-base:1.0.0 +MAINTAINER "Jonathan Brown" ENV container=docker VOLUME ["/sys/fs/cgroup"] LABEL vendor="VMware, Inc." -LABEL com.vmware.lightwave.version="1.2.0" +LABEL com.vmware.lightwave.version="1.3.1" EXPOSE 22 53/udp 53 88/udp 88 389 443 636 2012 2014 2015 2020 ENTRYPOINT ["/lightwave-init"] # Build hook -RUN tdnf update --refresh -y \ - tdnf \ - rpm-4.11.2 \ - pcre-8.39 \ - openssl-1.0.2j \ - expat-2.2.0 \ - curl-7.51.0 \ - bzip2-1.0.6 && \ - tdnf install -y \ - apache-tomcat-8.5.8 \ - boost-1.60.0 \ - commons-daemon-1.0.15 \ - likewise-open-6.2.11 \ - openjre-1.8.0.112 \ - procps-ng-3.3.11 \ - sed-4.2.2 \ - jansson-2.9 \ - vmware-lightwave-server-1.2.0 && \ - rpm -e --nodeps systemd && \ - mkdir -p /var/run/sshd && \ - chmod -rx /var/run/sshd && \ - rm -rf /usr/share/doc/* && \ - rm -rf /usr/share/man/* && \ - rm -rf /usr/include/* && \ +RUN tdnf install -y lightwave-1.3.1 && \ /opt/likewise/sbin/lwsmd --start-as-daemon && \ /opt/likewise/bin/lwregshell set_value "[HKEY_THIS_MACHINE\\Services\\vmafd]" \ Arguments "/opt/vmware/sbin/vmafdd -c" && \ @@ -42,5 +19,12 @@ RUN tdnf update --refresh -y \ /opt/likewise/bin/lwregshell set_value "[HKEY_THIS_MACHINE\\Services\\vmdir]" \ Arguments "/opt/vmware/sbin/vmdird -l 0 -f /opt/vmware/share/config/vmdirschema.ldif" && \ /opt/likewise/bin/lwregshell set_value "[HKEY_THIS_MACHINE\\Services\\vmdns]" \ - Arguments "/opt/vmware/sbin/vmdnsd" + Arguments "/opt/vmware/sbin/vmdnsd" && \ + rpm -e --nodeps systemd && \ + rpm -e createrepo && \ + rm -rf /usr/share/doc/* && \ + rm -rf /usr/share/man/* && \ + rm -rf /usr/include/* && \ + rm -rf /tmp/vmware/lightwave + ADD lightwave-init / diff --git a/support/docker/lightwave-init b/support/docker/sts/lightwave-init similarity index 100% rename from support/docker/lightwave-init rename to support/docker/sts/lightwave-init diff --git a/support/make/makedefs.mk b/support/make/makedefs.mk index 53d0fa460..af1881ade 100644 --- a/support/make/makedefs.mk +++ b/support/make/makedefs.mk @@ -20,7 +20,7 @@ LW_SERVER_PKGDIR=$(LW_SERVER_SRCROOT)/rpmbuild/RPMS/$(ARCH) LW_SERVER_MAJOR_VER=1 LW_SERVER_MINOR_VER=3 LW_SERVER_RELEASE_VER=1 -LW_SERVER_PATCH_VER=6 +LW_SERVER_PATCH_VER=7 LW_SERVER_VERSION=$(LW_SERVER_MAJOR_VER).$(LW_SERVER_MINOR_VER).$(LW_SERVER_RELEASE_VER)-$(LW_SERVER_PATCH_VER) LW_SERVER_PKG_NAME=vmware-lightwave-server LW_SERVER_RPM=$(LW_SERVER_PKG_NAME)-$(LW_SERVER_VERSION).$(ARCH).rpm diff --git a/support/scripts/clean.sh b/support/scripts/clean.sh index 5b9d8b9aa..ef31fd83c 100755 --- a/support/scripts/clean.sh +++ b/support/scripts/clean.sh @@ -8,6 +8,7 @@ cd $PROJECT_ROOT/build && \ /bin/rm -rf `find $PROJECT_ROOT -name Makefile.in` /bin/rm -rf config \ + docker \ include \ lwraft \ rpmbuild \ diff --git a/support/scripts/prep-container-build.sh b/support/scripts/prep-container-build.sh new file mode 100755 index 000000000..b739bc77d --- /dev/null +++ b/support/scripts/prep-container-build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +PROJECT_ROOT=$(pwd) + +DOCKER_ROOT=$PROJECT_ROOT/build/docker +DOCKER_SRC_ROOT=$PROJECT_ROOT/support/docker/sts + +mkdir -p $DOCKER_ROOT + +rm -rf $DOCKER_ROOT/* + +cp -r $PROJECT_ROOT/build/rpmbuild/RPMS/x86_64 $DOCKER_ROOT + +cp $DOCKER_SRC_ROOT/lightwave-init $DOCKER_ROOT +cp $DOCKER_SRC_ROOT/Dockerfile $DOCKER_ROOT + +# modify Dockerfile to use local lightwave yum repository + +# +# Assumes that sed and createrepo are already installed +# +tmpfile=$(mktemp /tmp/lw.XXXXXX) +cat >$tmpfile < ENV container=docker ENV GOROOT=/usr/lib/golang -RUN tdnf update -y --refresh tdnf && \ +RUN tdnf update -y --refresh rpm-4.13.0.1 tdnf && \ tdnf makecache && \ - tdnf update -y rpm && \ - tdnf install -y sed && \ - tdnf install -y procps-ng && \ - tdnf install -y shadow && \ - tdnf install -y binutils && \ - tdnf install -y make && \ - tdnf install -y gawk && \ - tdnf install -y autoconf && \ - tdnf install -y automake && \ - tdnf install -y libtool && \ - tdnf install -y gcc && \ - tdnf install -y glibc-devel && \ - tdnf install -y linux-api-headers && \ - tdnf install -y util-linux-devel && \ - tdnf install -y e2fsprogs-devel && \ - tdnf install -y rpm-build && \ - tdnf install -y rpm-devel && \ - tdnf install -y openjdk && \ - tdnf install -y apache-maven && \ - tdnf install -y apache-ant && \ - tdnf install -y ant-contrib && \ - tdnf install -y jaxws-ri && \ - tdnf install -y python2-devel && \ - tdnf install -y apache-tomcat && \ - tdnf install -y boost-devel && \ - tdnf install -y jansson-devel && \ + tdnf update -y tdnf && \ + tdnf install -y sed-4.2.2 && \ + tdnf install -y procps-ng-3.3.11 && \ + tdnf install -y shadow-4.2.1 && \ + tdnf install -y binutils-2.29.1 && \ + tdnf install -y make-4.1 && \ + tdnf install -y gawk-4.1.3 && \ + tdnf install -y autoconf-2.69 && \ + tdnf install -y automake-1.15 && \ + tdnf install -y libtool-2.4.6 && \ + tdnf install -y gcc-5.3.0 && \ + tdnf install -y glibc-devel-2.22 && \ + tdnf install -y linux-api-headers-4.4.88 && \ + tdnf install -y util-linux-devel-2.27.1 && \ + tdnf install -y e2fsprogs-devel-1.42.13 && \ + tdnf install -y rpm-build-4.13.0.1 && \ + tdnf install -y rpm-devel-4.13.0.1 && \ + tdnf install -y openjdk-1.8.0.141 && \ + tdnf install -y apache-maven-3.3.9 && \ + tdnf install -y apache-ant-1.9.6 && \ + tdnf install -y ant-contrib-1.0b3 && \ + tdnf install -y jaxws-ri-2.2.10 && \ + tdnf install -y python2-devel-2.7.13 && \ + tdnf install -y apache-tomcat-8.5.23 && \ + tdnf install -y boost-devel-1.60.0 && \ + tdnf install -y jansson-devel-2.9 && \ tdnf install -y openssl-devel && \ - tdnf install -y likewise-open-devel && \ - tdnf install -y copenapi-devel && \ - tdnf install -y c-rest-engine-devel && \ - tdnf install -y go && \ + tdnf install -y likewise-open-devel-6.2.11 && \ + tdnf install -y copenapi-devel-0.0.1 && \ + tdnf install -y c-rest-engine-devel-1.1 && \ + tdnf install -y go-1.8.1 && \ echo 'ALL ALL=NOPASSWD: ALL' >>/etc/sudoers && \ chmod -R o+r /opt/likewise/include diff --git a/vmca/common/misc.c b/vmca/common/misc.c index f406bf688..7d22efc31 100644 --- a/vmca/common/misc.c +++ b/vmca/common/misc.c @@ -515,6 +515,60 @@ VMCAGetRegKeyValue( } #endif +#ifndef _WIN32 +DWORD +VMCAGetRegKeyValueDword( + PCSTR pszConfigParamKeyPath, + PCSTR pszKey, + PDWORD pdwValue, + DWORD dwDefaultValue + ) +{ + DWORD dwError = 0; + DWORD dwValue = 0; + REG_DATA_TYPE RegType = 0; + + if (pszConfigParamKeyPath == NULL || pszKey == NULL || pdwValue == NULL) + { + dwError = ERROR_INVALID_PARAMETER; + BAIL_ON_VMCA_ERROR(dwError); + } + + VMCA_LOG_VERBOSE("Reading Reg: %s", pszKey); + + *pdwValue = dwDefaultValue; + + dwError = RegUtilGetValue( + NULL, + HKEY_THIS_MACHINE, + NULL, + pszConfigParamKeyPath, + pszKey, + &RegType, + (PVOID*)&dwValue, + NULL); + BAIL_ON_VMCA_ERROR(dwError); + + if (RegType != REG_DWORD) + { + dwError = ERROR_INVALID_PARAMETER; + BAIL_ON_VMCA_ERROR(dwError); + } + + *pdwValue = dwValue; + +cleanup: + return dwError; + +error: + VMCA_LOG_VERBOSE( + "VmDirGetRegKeyValueDword failed with error (%u)(%s)", + dwError, + VMCA_SAFE_STRING(pszKey)); + goto cleanup; +} +#endif + DWORD VMCAGetSharedSecret( PWSTR* ppszSharedSecret diff --git a/vmca/service/defines.h b/vmca/service/defines.h index e7ab55f26..7683bebd1 100644 --- a/vmca/service/defines.h +++ b/vmca/service/defines.h @@ -221,14 +221,23 @@ typedef DWORD VMCA_FUNC_LEVEL; } // C REST ENGINE CONFIG VALUES -#define VMCARESTSSLCERT "/root/mycert.pem" -#define VMCARESTSSLKEY "/root/mycert.pem" -#define VMCAHTTPPORT "7777p" -#define VMCAHTTPSPORT "7778" -#define VMCAHTTPDEBUGLOGFILE VMCA_LOG_DIR VMCA_PATH_SEPARATOR_STR "vmca-rest-http.log" -#define VMCAHTTPSDEBUGLOGFILE VMCA_LOG_DIR VMCA_PATH_SEPARATOR_STR "vmca-rest-https.log" -#define VMCARESTCLIENTCNT "5" -#define VMCARESTWORKERTHCNT "5" +#define VMCA_REST_SSL_CERT "/root/mycert.pem" +#define VMCA_REST_SSL_KEY "/root/mycert.pem" +#define VMCA_HTTP_PORT_STR "7777" +#define VMCA_HTTPS_PORT_STR "7778" +#define VMCA_HTTP_PORT_NUM 7777 +#define VMCA_HTTPS_PORT_NUM 7778 +#define VMCA_HTTP_DEBUG_LOGFILE VMCA_LOG_DIR VMCA_PATH_SEPARATOR_STR "vmca-rest-http.log" +#define VMCA_HTTPS_DEBUG_LOGFILE VMCA_LOG_DIR VMCA_PATH_SEPARATOR_STR "vmca-rest-https.log" +#define VMCA_REST_CLIENT_CNT 5 +#define VMCA_REST_WORKER_TH_CNT 5 + +#define VMCA_REST_CONN_TIMEOUT_SEC 30 +#define VMCA_MAX_DATA_PER_CONN_MB 25 +#define VMCA_HTTP_DAEMON_NAME "vmcad-http"; +#define VMCA_HTTPS_DAEMON_NAME "vmcad-https"; +#define VMCA_REST_STOP_TIMEOUT_SEC 10 + #define VMCARESTMAXPAYLOADLENGTH 4096 //Rest port config diff --git a/vmca/service/restserviceinit.c b/vmca/service/restserviceinit.c index 159353a26..d0abc6545 100644 --- a/vmca/service/restserviceinit.c +++ b/vmca/service/restserviceinit.c @@ -61,11 +61,9 @@ _VMCAHttpsServiceShutdown( ); static -DWORD -_VMCAGetRestRegPort( - PSTR pszPortRegKeyStr, - PSTR pszDefaultPort, - PSTR* ppszRegPort +VOID +_VMCARestFreeHandle( + PVMREST_HANDLE pHandle ); DWORD @@ -114,31 +112,41 @@ _VMCAHttpServiceStartup( DWORD dwError = 0; DWORD iter = 0; DWORD endPointCnt = 0; - PSTR pszPort = NULL; + DWORD dwPort = 0; REST_CONF config = {0}; PREST_PROCESSOR pHandlers = &sVmcaRestHandlers; + PVMREST_HANDLE pHTTPHandle = NULL; - dwError = _VMCAGetRestRegPort( - VMCA_HTTP_PORT_REG_KEY, - VMCAHTTPPORT, - &pszPort - ); - BAIL_ON_VMCA_ERROR(dwError); + (VOID)VMCAGetRegKeyValueDword( + VMCA_KEY_PARAMETERS,//VMCA_CONFIG_PARAMETER_KEY_PATH, + VMCA_HTTP_PORT_REG_KEY, + &dwPort, + VMCA_HTTP_PORT_NUM + ); - // empty port string indicates don't start corresponding service - if (IsNullOrEmptyString(pszPort)) + // port value '0' indicates don't start HTTP service + if (dwPort == 0) { goto cleanup; } - config.pSSLCertificate = VMCARESTSSLCERT; - config.pSSLKey = VMCARESTSSLKEY; - config.pServerPort = pszPort; - config.pDebugLogFile = VMCAHTTPDEBUGLOGFILE; - config.pClientCount = VMCARESTCLIENTCNT; - config.pMaxWorkerThread = VMCARESTWORKERTHCNT; - - dwError = VmRESTInit(&config, NULL, &gpVMCAHTTPHandle); + config.serverPort = dwPort; + config.connTimeoutSec = VMCA_REST_CONN_TIMEOUT_SEC; + config.maxDataPerConnMB = VMCA_MAX_DATA_PER_CONN_MB; + config.pSSLContext = NULL; + config.nWorkerThr = VMCA_REST_WORKER_TH_CNT; + config.nClientCnt = VMCA_REST_CLIENT_CNT; + config.SSLCtxOptionsFlag = 0; + config.pszSSLCertificate = NULL; + config.pszSSLKey = NULL; + config.pszSSLCipherList = NULL; + config.pszDebugLogFile = VMCA_HTTP_DEBUG_LOGFILE; + config.pszDaemonName = VMCA_HTTP_DAEMON_NAME; + config.isSecure = FALSE; + config.useSysLog = TRUE; + config.debugLogLevel = VMREST_LOG_LEVEL_INFO; + + dwError = VmRESTInit(&config, &pHTTPHandle); BAIL_ON_VMREST_ERROR(dwError); endPointCnt = ARRAY_SIZE(restEndPoints); @@ -146,22 +154,23 @@ _VMCAHttpServiceStartup( for (iter = 0; iter < endPointCnt; iter++) { dwError = VmRESTRegisterHandler( - gpVMCAHTTPHandle, + pHTTPHandle, restEndPoints[iter], pHandlers, NULL); BAIL_ON_VMREST_ERROR(dwError); } - dwError = VmRESTStart(gpVMCAHTTPHandle); + dwError = VmRESTStart(pHTTPHandle); BAIL_ON_VMREST_ERROR(dwError); + gpVMCAHTTPHandle = pHTTPHandle; + cleanup: - VMCA_SAFE_FREE_MEMORY(pszPort); return dwError; error: - _VMCAHttpServiceShutdown(); + _VMCARestFreeHandle(pHTTPHandle); VMCA_LOG_ERROR("%s: failure while starting REST HTTP service, error: %d", __FUNCTION__, dwError); goto cleanup; } @@ -178,40 +187,50 @@ _VMCAHttpsServiceStartup( REST_CONF config = {0}; PSTR pszCert = NULL; PSTR pszKey = NULL; - PSTR pszPort = NULL; + DWORD dwPort = 0; PREST_PROCESSOR pHandlers = &sVmcaRestHandlers; + PVMREST_HANDLE pHTTPSHandle = NULL; - dwError = _VMCAGetRestRegPort( - VMCA_HTTPS_PORT_REG_KEY, - VMCAHTTPSPORT, - &pszPort - ); - BAIL_ON_VMCA_ERROR(dwError); + (VOID)VMCAGetRegKeyValueDword( + VMCA_KEY_PARAMETERS,//VMCA_CONFIG_PARAMETER_KEY_PATH, + VMCA_HTTPS_PORT_REG_KEY, + &dwPort, + VMCA_HTTPS_PORT_NUM + ); - // empty port string indicates don't start corresponding service - if (IsNullOrEmptyString(pszPort)) + // port value '0' indicates don't start HTTPS service + if (dwPort == 0) { goto cleanup; } - config.pSSLCertificate = NULL; - config.pSSLKey = NULL; - config.pServerPort = pszPort; - config.pDebugLogFile = VMCAHTTPSDEBUGLOGFILE; - config.pClientCount = VMCARESTCLIENTCNT; - config.pMaxWorkerThread = VMCARESTWORKERTHCNT; + config.serverPort = dwPort; + config.connTimeoutSec = VMCA_REST_CONN_TIMEOUT_SEC; + config.maxDataPerConnMB = VMCA_MAX_DATA_PER_CONN_MB; + config.pSSLContext = NULL; + config.nWorkerThr = VMCA_REST_WORKER_TH_CNT; + config.nClientCnt = VMCA_REST_CLIENT_CNT; + config.SSLCtxOptionsFlag = 0; + config.pszSSLCertificate = NULL; + config.pszSSLKey = NULL; + config.pszSSLCipherList = NULL; + config.pszDebugLogFile = VMCA_HTTPS_DEBUG_LOGFILE; + config.pszDaemonName = VMCA_HTTPS_DAEMON_NAME; + config.isSecure = TRUE; + config.useSysLog = TRUE; + config.debugLogLevel = VMREST_LOG_LEVEL_INFO; //Get Certificate and Key from VECS and Set it to Rest Engine dwError = VMCAGetVecsMachineCert(&pszCert, &pszKey); BAIL_ON_VMREST_ERROR(dwError); - dwError = VmRESTInit(&config, NULL, &gpVMCAHTTPSHandle); + dwError = VmRESTInit(&config, &pHTTPSHandle); BAIL_ON_VMREST_ERROR(dwError); - dwError = VmRESTSetSSLInfo(gpVMCAHTTPSHandle, pszCert, VMCAStringLenA(pszCert)+1, SSL_DATA_TYPE_CERT); + dwError = VmRESTSetSSLInfo(pHTTPSHandle, pszCert, VMCAStringLenA(pszCert)+1, SSL_DATA_TYPE_CERT); BAIL_ON_VMREST_ERROR(dwError); - dwError = VmRESTSetSSLInfo(gpVMCAHTTPSHandle, pszKey, VMCAStringLenA(pszKey)+1, SSL_DATA_TYPE_KEY); + dwError = VmRESTSetSSLInfo(pHTTPSHandle, pszKey, VMCAStringLenA(pszKey)+1, SSL_DATA_TYPE_KEY); BAIL_ON_VMREST_ERROR(dwError); endPointCnt = ARRAY_SIZE(restEndPoints); @@ -219,24 +238,25 @@ _VMCAHttpsServiceStartup( for (iter = 0; iter < endPointCnt; iter++) { dwError = VmRESTRegisterHandler( - gpVMCAHTTPSHandle, + pHTTPSHandle, restEndPoints[iter], pHandlers, NULL); BAIL_ON_VMREST_ERROR(dwError); } - dwError = VmRESTStart(gpVMCAHTTPSHandle); + dwError = VmRESTStart(pHTTPSHandle); BAIL_ON_VMREST_ERROR(dwError); + gpVMCAHTTPSHandle = pHTTPSHandle; + cleanup: VMCA_SAFE_FREE_MEMORY(pszCert); VMCA_SAFE_FREE_MEMORY(pszKey); - VMCA_SAFE_FREE_MEMORY(pszPort); return dwError; error: - _VMCAHttpsServiceShutdown(); + _VMCARestFreeHandle(pHTTPSHandle); VMCA_LOG_ERROR("%s: failure while starting REST HTTPS service, error: %d", __FUNCTION__, dwError); goto cleanup; } @@ -247,23 +267,11 @@ _VMCAHttpServiceShutdown( VOID ) { - DWORD iter = 0; - DWORD endPointCnt = 0; - VMCA_LOG_INFO("%s: starting http rest server shutdown:", __FUNCTION__); - if (gpVMCAHTTPHandle) - { - VmRESTStop(gpVMCAHTTPHandle); - endPointCnt = ARRAY_SIZE(restEndPoints); - for (iter = 0; iter < endPointCnt; iter++) - { - (VOID)VmRESTUnRegisterHandler( - gpVMCAHTTPHandle, - restEndPoints[iter]); - } - VmRESTShutdown(gpVMCAHTTPHandle); - } + + _VMCARestFreeHandle(gpVMCAHTTPHandle); gpVMCAHTTPHandle = NULL; + VMCA_LOG_INFO("%s: completed http rest server shutdown:", __FUNCTION__); } @@ -273,81 +281,49 @@ _VMCAHttpsServiceShutdown( VOID ) { - DWORD iter = 0; - DWORD endPointCnt = 0; - VMCA_LOG_INFO("%s: starting https rest server shutdown:", __FUNCTION__); - if (gpVMCAHTTPSHandle) - { - VmRESTStop(gpVMCAHTTPSHandle); - endPointCnt = ARRAY_SIZE(restEndPoints); - for (iter = 0; iter < endPointCnt; iter++) - { - (VOID)VmRESTUnRegisterHandler( - gpVMCAHTTPSHandle, - restEndPoints[iter]); - } - VmRESTShutdown(gpVMCAHTTPSHandle); - } + + _VMCARestFreeHandle(gpVMCAHTTPSHandle); gpVMCAHTTPSHandle = NULL; + VMCA_LOG_INFO("%s: completed https rest server shutdown:", __FUNCTION__); } static -DWORD -_VMCAGetRestRegPort( - PSTR pszPortRegKeyStr, - PSTR pszDefaultPort, - PSTR* ppszRegPort +VOID +_VMCARestFreeHandle( + PVMREST_HANDLE pHandle ) { + DWORD iter = 0; DWORD dwError = 0; - PSTR pszPort = NULL; + DWORD endPointCnt = 0; - if (ppszRegPort == NULL) + if (pHandle) { - dwError = VMCA_ARGUMENT_ERROR; - BAIL_ON_VMCA_ERROR(dwError); - } - - dwError = VMCAAllocateMemory( - VMCA_REST_PORT_STR_MAX_SIZE, - (PVOID*)&pszPort - ); - BAIL_ON_VMCA_ERROR(dwError); - - dwError = VMCAGetRegKeyValue( - VMCA_KEY_PARAMETERS, - pszPortRegKeyStr, - pszPort, - VMCA_REST_PORT_STR_MAX_SIZE - ); + /* + * REST library have detached threads, maximum time out specified is the max time + * allowed for the threads to finish their execution. + * If finished early, it will return success. + * If not able to finish in specified time, failure will be returned + */ + dwError = VmRESTStop(pHandle, VMCA_REST_STOP_TIMEOUT_SEC); + + if (dwError != 0) + { + VMCA_LOG_WARNING("%s: rest server stop error:%d", __FUNCTION__, dwError); + } - if (dwError != 0) - { - //assign default value - VMCAStringNCpyA( - pszPort, - VMCAStringLenA(pszDefaultPort)+1, - pszDefaultPort, - VMCAStringLenA(pszDefaultPort)+1 - ); - //not a failure - dwError = 0; + endPointCnt = ARRAY_SIZE(restEndPoints); + for (iter = 0; iter < endPointCnt; iter++) + { + (VOID)VmRESTUnRegisterHandler( + pHandle, + restEndPoints[iter]); + } + VmRESTShutdown(pHandle); } - - //transfer ownership - *ppszRegPort = pszPort; - pszPort = NULL; - -cleanup: - VMCA_SAFE_FREE_MEMORY(pszPort); - return dwError; - -error: - VMCA_LOG_ERROR("%s: failed error: %d", __FUNCTION__, dwError); - goto cleanup; - } + #endif #endif diff --git a/vmca/service/vmcaHTTPHandlers.c b/vmca/service/vmcaHTTPHandlers.c index 6f966c54b..7a3bd4312 100644 --- a/vmca/service/vmcaHTTPHandlers.c +++ b/vmca/service/vmcaHTTPHandlers.c @@ -87,7 +87,8 @@ VMCARESTParseHttpHeader( BAIL_ON_VMREST_ERROR(dwError); pVMCARequest->pszMethod = pszBuff; - dwError = VmRESTGetHttpURI(pRESTRequest, &pszBuff); + // TRUE - Request c-rest-engine to decode the URI + dwError = VmRESTGetHttpURI(pRESTRequest, TRUE, &pszBuff); BAIL_ON_VMREST_ERROR(dwError); pVMCARequest->pszUri = pszBuff; diff --git a/vmdir/client/client.c b/vmdir/client/client.c index d972732fd..035b1fc71 100755 --- a/vmdir/client/client.c +++ b/vmdir/client/client.c @@ -148,10 +148,12 @@ VmDirRefreshActPassword( BAIL_ON_VMDIR_ERROR(dwError); } - dwError = VmDirSafeLDAPBind( &pLD, - pszHost, - pszActUPN, - pszActPassword); + dwError = VmDirSafeLDAPBindExt1( + &pLD, + pszHost, + pszActUPN, + pszActPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirDomainNameToDN( pszDomain, &pszDomainDN); @@ -2871,10 +2873,12 @@ VmDirGetServers( dwError = VmDirStringPrintFA( bufUPN, sizeof(bufUPN)-1, "%s@%s", pszUserName, pszDomain); BAIL_ON_VMDIR_ERROR(dwError); - dwError = VmDirSafeLDAPBind( &pLd, - pszServerName, - bufUPN, - pszPassword); + dwError = VmDirSafeLDAPBindExt1( + &pLd, + pszServerName, + bufUPN, + pszPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); //get all vmdir servers in the forest. @@ -5162,10 +5166,12 @@ _VmDirCreateServerPLD( dwError = VmDirAllocateStringPrintf( &pszUPN, "%s@%s", pszUserName, pszDomain ); BAIL_ON_VMDIR_ERROR(dwError); - dwError = VmDirSafeLDAPBind( &pLD, - pszServerName, - pszUPN, - pszPassword); + dwError = VmDirSafeLDAPBindExt1( + &pLD, + pszServerName, + pszUPN, + pszPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); *ppLD = pLD; @@ -5260,10 +5266,12 @@ _VmDirModDcPassword( DWORD dwError = 0; LDAP* pLD = NULL; - dwError = VmDirSafeLDAPBind(&pLD, - pszHostName, - pszUPN, - pszPassword); + dwError = VmDirSafeLDAPBindExt1( + &pLD, + pszHostName, + pszUPN, + pszPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirLdapModReplaceAttribute(pLD, @@ -5312,11 +5320,12 @@ VmDirGetDomainFunctionalLevel( pszUserName, pszDomainName); BAIL_ON_VMDIR_ERROR(dwError); - dwError = VmDirSafeLDAPBind( + dwError = VmDirSafeLDAPBindExt1( &pLd, pszHostName, bufUPN, - pszPassword ); + pszPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirGetDomainFuncLvlInternal( @@ -5399,11 +5408,12 @@ VmDirSetDomainFunctionalLevel( } - dwError = VmDirSafeLDAPBind( + dwError = VmDirSafeLDAPBindExt1( &pLd, pszHostName, bufUPN, - pszPassword); + pszPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); // Do not allow DFL downgrade. @@ -5683,10 +5693,12 @@ VmDirPSCVersion( pszUserName, pszDomainName); BAIL_ON_VMDIR_ERROR(dwError); - dwError = VmDirSafeLDAPBind( &pLd, - pszHostName, - bufUPN, - pszPassword); + dwError = VmDirSafeLDAPBindExt1( + &pLd, + pszHostName, + bufUPN, + pszPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirGetPSCVersionInternal( diff --git a/vmdir/client/ldaputil.c b/vmdir/client/ldaputil.c index 046dec76b..492d14d44 100644 --- a/vmdir/client/ldaputil.c +++ b/vmdir/client/ldaputil.c @@ -649,10 +649,12 @@ VmDirConnectLDAPServer( dwError = VmDirAllocateStringPrintf(&pszUPN, "%s@%s", pszUserName, pszDomain); BAIL_ON_VMDIR_ERROR(dwError); - dwError = VmDirSafeLDAPBind( &pLocalLd, - pszHostName, - pszUPN, - pszPassword); + dwError = VmDirSafeLDAPBindExt1( + &pLocalLd, + pszHostName, + pszUPN, + pszPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); *ppLd = pLocalLd; diff --git a/vmdir/client/redundancy.c b/vmdir/client/redundancy.c index 27c30e596..8f606533a 100644 --- a/vmdir/client/redundancy.c +++ b/vmdir/client/redundancy.c @@ -766,10 +766,12 @@ _VmDirGetServersOnSite( dwError = VmDirStringPrintFA( bufUPN, sizeof(bufUPN)-1, "%s@%s", pszUserName, pszDomain); BAIL_ON_VMDIR_ERROR(dwError); - dwError = VmDirSafeLDAPBind( &pLd, - pszServerName, - bufUPN, - pszPassword); + dwError = VmDirSafeLDAPBindExt1( + &pLd, + pszServerName, + bufUPN, + pszPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); //get all vmdir servers in the site. diff --git a/vmdir/client/users.c b/vmdir/client/users.c index f82a38b3a..3a88cd82a 100644 --- a/vmdir/client/users.c +++ b/vmdir/client/users.c @@ -290,11 +290,12 @@ VmDirSetPassword( BAIL_ON_VMDIR_ERROR(dwError); } - dwError = VmDirSafeLDAPBind( + dwError = VmDirSafeLDAPBindExt1( &pLd, pszHostName, pszAdminUPN, - pszAdminPassword); + pszAdminPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); dwError = _VmDirFindUserDN( @@ -353,11 +354,12 @@ VmDirChangePassword( BAIL_ON_VMDIR_ERROR(dwError); } - dwError = VmDirSafeLDAPBind( + dwError = VmDirSafeLDAPBindExt1( &pLd, pszHostName, pszUserUPN, - pszOldPassword); + pszOldPassword, + MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); dwError = _VmDirFindUserDN( diff --git a/vmdir/common/logging.c b/vmdir/common/logging.c index ae464f441..3a80ca0e3 100644 --- a/vmdir/common/logging.c +++ b/vmdir/common/logging.c @@ -187,7 +187,23 @@ VmDirLogInitialize( goto cleanup; } +BOOLEAN +VmDirLogLevelAndMaskTest( + VMDIR_LOG_LEVEL iLevel, + ULONG iMask + ) +{ + BOOLEAN bRtn = FALSE; + if (_gpVmDirLogCtx && + _gpVmDirLogCtx->iLogLevel >= iLevel && + iMask & _gpVmDirLogCtx->iLogMask) + { + bRtn = TRUE; + } + + return bRtn; +} void VmDirLog1( diff --git a/vmdir/include/public/vmdir.h b/vmdir/include/public/vmdir.h index 95bc8d860..55b1bfa0c 100644 --- a/vmdir/include/public/vmdir.h +++ b/vmdir/include/public/vmdir.h @@ -46,7 +46,7 @@ extern "C" { #define DEFAULT_LDAPS_PORT_STR "636" #define DEFAULT_HTTP_PORT_NUM 7477 -#define DEFAULT_HTTP_PORT_STR "7477p" +#define DEFAULT_HTTP_PORT_STR "7477" #define DEFAULT_HTTPS_PORT_NUM 7478 #define DEFAULT_HTTPS_PORT_STR "7478" diff --git a/vmdir/include/public/vmdirtypes.h b/vmdir/include/public/vmdirtypes.h index 7eda506db..39cd0ab40 100755 --- a/vmdir/include/public/vmdirtypes.h +++ b/vmdir/include/public/vmdirtypes.h @@ -391,6 +391,22 @@ typedef struct _VMDIR_SUPERLOG_TABLE PVMDIR_SUPERLOG_TABLE_ROW rows; } VMDIR_SUPERLOG_TABLE, *PVMDIR_SUPERLOG_TABLE; +#ifndef MDB_STATE_OP +#define MDB_STATE_OP + +typedef enum MDB_state_op { + MDB_STATE_CLEAR = 0, + MDB_STATE_READONLY, + MDB_STATE_KEEPXLOGS, + MDB_STATE_GETXLOGNUM +} MDB_state_op; + +#endif + +#define VMDIR_MDB_DATA_FILE_NAME "data.mdb" +#define VMDIR_MDB_LOCK_FILE_NAME "lock.mdb" +#define VMDIR_MDB_XLOGS_DIR_NAME "xlogs" + #ifdef VMDIR_ENABLE_PAC typedef struct _RPC_UNICODE_STRING { diff --git a/vmdir/include/vmdircommon.h b/vmdir/include/vmdircommon.h index b49664c84..7f9f521d3 100644 --- a/vmdir/include/vmdircommon.h +++ b/vmdir/include/vmdircommon.h @@ -582,6 +582,13 @@ VmDirLog1( const char* fmt, ...); +LOGGING_API +BOOLEAN +VmDirLogLevelAndMaskTest( + VMDIR_LOG_LEVEL iLevel, + ULONG iMask + ); + LOGGING_API DWORD VmDirLogInitialize( @@ -966,6 +973,12 @@ typedef enum // #define VMDIR_REG_KEY_TOMBSTONE_REAPING_FREQ_IN_SEC "TombstoneReapingThreadFreqInSec" +#define VMDIR_REG_KEY_MDB_ENABLE_WAL "MdbEnableWal" +#define VMDIR_REG_KEY_MDB_CHKPT_INTERVAL "MdbChkptInterval" +#define VMDIR_REG_KEY_MDB_CHKPT_INTERVAL_MIN 1 +#define VMDIR_REG_KEY_MDB_CHKPT_INTERVAL_MAX 180 +#define VMDIR_REG_KEY_MDB_CHKPT_INTERVAL_DEFAULT 30 + #ifdef _WIN32 #define VMDIR_DEFAULT_KRB5_CONF "C:\\ProgramData\\MIT\\Kerberos5\\krb5.ini" #else @@ -1495,6 +1508,16 @@ VmDirGetRegKeyValueQword( PINT64 pi64Value ); +DWORD +VmDirGetMdbWalEnable( + BOOLEAN *pbMdbEnableWal + ); + +DWORD +VmDirGetMdbChkptInterval( + DWORD *pdwMdbChkptInterval + ); + DWORD VmDirLoadLibrary( PCSTR pszLibPath, diff --git a/vmdir/server/include/srvcommon.h b/vmdir/server/include/srvcommon.h index bd625e431..64b252279 100644 --- a/vmdir/server/include/srvcommon.h +++ b/vmdir/server/include/srvcommon.h @@ -131,6 +131,8 @@ extern "C" { (gVmdirServerGlobals.dwDomainFunctionalLevel >= \ VDIR_DFL_CONCURRENT_ATTR_VALUE_UPDATE) +#define MAX_NUM_CONTENT_LOG 256 + // Keys for backend funtion pfnBEStrkeyGet/SetValues to access attribute IDs #define ATTR_ID_MAP_KEY "1VmdirAttrIDToNameTb" diff --git a/vmdir/server/include/vmdirserver.h b/vmdir/server/include/vmdirserver.h index a46d2b04c..c2389a6b8 100644 --- a/vmdir/server/include/vmdirserver.h +++ b/vmdir/server/include/vmdirserver.h @@ -145,8 +145,8 @@ typedef struct _VMDIR_GLOBALS DWORD dwLdapConnectPorts; PDWORD pdwLdapsConnectPorts; DWORD dwLdapsConnectPorts; - PSTR pszHTTPListenPort; - PSTR pszHTTPSListenPort; + DWORD dwHTTPListenPort; + DWORD dwHTTPSListenPort; DWORD dwLdapRecvTimeoutSec; DWORD dwLdapConnectTimeoutSec; @@ -214,6 +214,8 @@ typedef struct _VMDIR_GLOBALS DWORD dwCopyDbBlockWriteInSec; // Collect stats for estimate elapsed time with database copy DWORD dwLdapWrites; + + SSL_CTX* gpVdirSslCtx; } VMDIR_GLOBALS, *PVMDIR_GLOBALSS; extern VMDIR_GLOBALS gVmdirGlobals; diff --git a/vmdir/server/ldap-head/defines.h b/vmdir/server/ldap-head/defines.h index 91166502a..adc2f3f1e 100644 --- a/vmdir/server/ldap-head/defines.h +++ b/vmdir/server/ldap-head/defines.h @@ -27,8 +27,6 @@ #define MAX_NUM_OF_SOCK_READ_RETRIES 20 -#define MAX_NUM_MOD_CONTENT_LOG 256 - #ifdef _WIN32 #define MAX_NUM_OF_BIND_PORT_RETRIES 30 diff --git a/vmdir/server/ldap-head/modify.c b/vmdir/server/ldap-head/modify.c index f8ddcdf2b..428d998d6 100644 --- a/vmdir/server/ldap-head/modify.c +++ b/vmdir/server/ldap-head/modify.c @@ -135,7 +135,7 @@ VmDirPerformModify( PCSTR pszLogValue = (0 == VmDirStringCompareA( pMod->attr.type.lberbv.bv_val, ATTR_USER_PASSWORD, FALSE)) ? "XXX" : pLberBerv[iCnt].bv_val; - if (iCnt < MAX_NUM_MOD_CONTENT_LOG) + if (iCnt < MAX_NUM_CONTENT_LOG) { VMDIR_LOG_INFO( VMDIR_LOG_MASK_ALL, "MOD %d,%s,%s: (%.*s)", ++dwModCount, @@ -144,7 +144,7 @@ VmDirPerformModify( VMDIR_MIN(pLberBerv[iCnt].bv_len, VMDIR_MAX_LOG_OUTPUT_LEN), VDIR_SAFE_STRING( pszLogValue )); } - else if (iCnt == MAX_NUM_MOD_CONTENT_LOG) + else if (iCnt == MAX_NUM_CONTENT_LOG) { VMDIR_LOG_INFO( VMDIR_LOG_MASK_ALL, "MOD %d,%s,%s: .... Total MOD %d)", ++dwModCount, diff --git a/vmdir/server/ldap-head/openssl.c b/vmdir/server/ldap-head/openssl.c index f89c8e9a3..a8d710b79 100644 --- a/vmdir/server/ldap-head/openssl.c +++ b/vmdir/server/ldap-head/openssl.c @@ -135,8 +135,6 @@ Sockbuf_IO opensslBerSockbufIO = { /////////////////////////////////////////////////////////////////////////////// Sockbuf_IO* gpVdirBerSockbufIOOpenssl = &opensslBerSockbufIO; -// initialized in VmDirOpensslInit during startup and used to create SSL* -static SSL_CTX* gpVdirSslCtx = NULL; /////////////////////////////////////////////////////////////////////////////// // ****************** shared static variables end ******************** /////////////////////////////////////////////////////////////////////////////// @@ -347,7 +345,7 @@ _VmDirInitSslCtx( } /* - * Initialize openssl libraries and create a default SSL_CTX - gpVdirSslCtx + * Initialize openssl libraries and create a default SSL_CTX - gVmdirGlobals.gpVdirSslCtx */ DWORD VmDirOpensslInit( @@ -437,7 +435,7 @@ VmDirOpensslInit( BAIL_ON_OPENSSL_ERROR( TRUE, "SSL_CTX_check_private_key"); } - gpVdirSslCtx = pSslCtx; + gVmdirGlobals.gpVdirSslCtx = pSslCtx; gVmdirOpensslGlobals.bSSLInitialized = TRUE; cleanup: @@ -472,9 +470,9 @@ VmDirOpensslShutdown( { DWORD dwSize = 0; - if (gpVdirSslCtx) + if (gVmdirGlobals.gpVdirSslCtx) { - SSL_CTX_free(gpVdirSslCtx); + SSL_CTX_free(gVmdirGlobals.gpVdirSslCtx); } ERR_remove_state(0); @@ -635,7 +633,7 @@ opensslSbSetup( pSbiod->sbiod_sb->sb_fd = *((int *)pArg); - pSsl = SSL_new(gpVdirSslCtx); + pSsl = SSL_new(gVmdirGlobals.gpVdirSslCtx); BAIL_ON_OPENSSL_ERROR( (pSsl == NULL), "SSL_new" ); #ifdef _WIN32 diff --git a/vmdir/server/mdb-store/Makefile.am b/vmdir/server/mdb-store/Makefile.am index 650c4b0c9..90cde43b9 100644 --- a/vmdir/server/mdb-store/Makefile.am +++ b/vmdir/server/mdb-store/Makefile.am @@ -35,7 +35,8 @@ libmdb_store_la_CPPFLAGS = \ -I$(top_srcdir)/vmdir/server/include \ -I$(top_srcdir)/vmmetrics/include/public \ @LW_INCLUDES@ \ - @OPENSSL_INCLUDES@ + @OPENSSL_INCLUDES@ \ + -D MDB_USE_PWRITEV libmdb_store_la_LDFLAGS = \ -static \ diff --git a/vmdir/server/mdb-store/init.c b/vmdir/server/mdb-store/init.c index 953dc2407..d203be0cb 100644 --- a/vmdir/server/mdb-store/init.c +++ b/vmdir/server/mdb-store/init.c @@ -636,7 +636,7 @@ MDBFreeMdbGlobals( */ DWORD VmDirSetMdbBackendState( - DWORD dwFileTransferState, + MDB_state_op op, DWORD *pdwLogNum, DWORD *pdwDbSizeMb, DWORD *pdwDbMapSizeMb, @@ -648,30 +648,35 @@ VmDirSetMdbBackendState( unsigned long dbSizeMb = 0L; unsigned long dbMapSizeMb = 0L; - if (dwFileTransferState < 0 || dwFileTransferState > 2) + if (op < MDB_STATE_CLEAR || op > MDB_STATE_GETXLOGNUM) { dwError = ERROR_INVALID_PARAMETER; BAIL_ON_VMDIR_ERROR(dwError); } - if (dwFileTransferState == 0) - { - VmDirdStateSet(VMDIRD_STATE_NORMAL); - } - else - { - VmDirdStateSet(VMDIRD_STATE_READ_ONLY); - } - *pdwLogNum = 0; *pdwDbSizeMb = 0; *pdwDbMapSizeMb = 0; - dwError = mdb_env_set_state(gVdirMdbGlobals.mdbEnv, dwFileTransferState, &lognum, &dbSizeMb, &dbMapSizeMb, pszDbPath, dwDbPathSize); + dwError = mdb_env_set_state(gVdirMdbGlobals.mdbEnv, op, &lognum, &dbSizeMb, &dbMapSizeMb, pszDbPath, dwDbPathSize); BAIL_ON_VMDIR_ERROR(dwError); *pdwLogNum = lognum; *pdwDbSizeMb = dbSizeMb; *pdwDbMapSizeMb = dbMapSizeMb; + if (op==MDB_STATE_CLEAR||op==MDB_STATE_READONLY||op==MDB_STATE_KEEPXLOGS) + { + //Log MDB state change event + if (op==MDB_STATE_KEEPXLOGS && lognum == 0) + { + VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, + "VmDirSetMdbBackendState: set MDB state to ReadOnly for request keepXlogs - MdbEnableWal disabled"); + } else + { + VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "VmDirSetMdbBackendState: set MDB state to %s", + op==MDB_STATE_CLEAR?"clear":(op==MDB_STATE_READONLY?"ReadOnly":(op==MDB_STATE_KEEPXLOGS?"keepXlogs":""))); + } + } + cleanup: return dwError; @@ -1005,6 +1010,7 @@ _VmDirOpenDbEnv() uint64_t db_max_mapsize = BE_MDB_ENV_MAX_MEM_MAPSIZE; DWORD db_max_size_mb = 0; PSTR pszLocalErrorMsg = NULL; + BOOLEAN bMdbWalEnable = FALSE; #ifndef _WIN32 const char *dbHomeDir = VMDIR_DB_DIR; char dbSnapshotDir[VMDIR_MAX_FILE_NAME_LEN] = {0}; @@ -1060,6 +1066,21 @@ _VmDirOpenDbEnv() //envFlags |= MDB_RDONLY need to open for read and write /* Open the environment. */ + //MDB NOWAL is the default mode and can be turned on with reg key MdbEnableWal set to 1 + dwError = VmDirGetMdbWalEnable(&bMdbWalEnable); + if (dwError) + { + bMdbWalEnable = FALSE; + dwError = 0; + } + + VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "%s: %s is set to %s", + __func__, VMDIR_REG_KEY_MDB_ENABLE_WAL, bMdbWalEnable?"True":"False"); + + if (bMdbWalEnable) + { + envFlags |= MDB_WAL; + } #ifndef _WIN32 oflags = O_RDWR; @@ -1177,6 +1198,7 @@ DWORD _VmdirCreateDbEnv(uint64_t db_max_mapsize) { DWORD dwError = 0; + DWORD db_chkpt_interval = 0; /* Create the environment */ dwError = mdb_env_create ( &gVdirMdbGlobals.mdbEnv ); @@ -1190,6 +1212,20 @@ _VmdirCreateDbEnv(uint64_t db_max_mapsize) dwError = mdb_env_set_maxdbs ( gVdirMdbGlobals.mdbEnv, BE_MDB_ENV_MAX_DBS ); BAIL_ON_VMDIR_ERROR( dwError ); + + dwError = VmDirGetMdbChkptInterval(&db_chkpt_interval); + if (dwError) + { + db_chkpt_interval = VMDIR_REG_KEY_MDB_CHKPT_INTERVAL_DEFAULT; + dwError = 0; + } + + VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "%s: %s is set to %d", + __func__, VMDIR_REG_KEY_MDB_CHKPT_INTERVAL, db_chkpt_interval); + + dwError = mdb_env_set_chkpt_interval(gVdirMdbGlobals.mdbEnv, db_chkpt_interval); + BAIL_ON_VMDIR_ERROR( dwError ); + cleanup: return dwError; diff --git a/vmdir/server/replication/firstreplcycle.c b/vmdir/server/replication/firstreplcycle.c index 3294375c5..e2e5e75db 100644 --- a/vmdir/server/replication/firstreplcycle.c +++ b/vmdir/server/replication/firstreplcycle.c @@ -48,7 +48,8 @@ static int _VmDirGetRemoteDBUsingRPC( PCSTR pszHostname, - PCSTR dbHomeDir); + PCSTR dbHomeDir, + BOOLEAN *pbHasXlog); static int @@ -62,7 +63,8 @@ _VmDirGetRemoteDBFileUsingRPC( static int _VmDirSwapDB( - PCSTR dbHomeDir); + PCSTR dbHomeDir, + BOOLEAN bHasXlog); static int @@ -99,6 +101,7 @@ VmDirFirstReplicationCycle( int retVal = LDAP_SUCCESS; PSTR pszLocalErrorMsg = NULL; BOOLEAN bWriteInvocationId = FALSE; + BOOLEAN bHasXlog = FALSE; #ifndef _WIN32 const char *dbHomeDir = VMDIR_DB_DIR; #else @@ -117,11 +120,11 @@ VmDirFirstReplicationCycle( assert( gFirstReplCycleMode == FIRST_REPL_CYCLE_MODE_COPY_DB ); - retVal = _VmDirGetRemoteDBUsingRPC(pszHostname, dbHomeDir); + retVal = _VmDirGetRemoteDBUsingRPC(pszHostname, dbHomeDir, &bHasXlog); BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), "VmDirFirstReplicationCycle: _VmDirGetRemoteDBUsingRPC() call failed with error: %d", retVal ); - retVal = _VmDirSwapDB(dbHomeDir); + retVal = _VmDirSwapDB(dbHomeDir, bHasXlog); BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), "VmDirFirstReplicationCycle: _VmDirSwapDB() call failed, error: %d.", retVal ); @@ -151,20 +154,25 @@ static int _VmDirGetRemoteDBUsingRPC( PCSTR pszHostname, - PCSTR dbHomeDir) + PCSTR dbHomeDir, + BOOLEAN *pbHasXlog) { DWORD retVal = 0; PSTR pszLocalErrorMsg = NULL; char dbRemoteFilename[VMDIR_MAX_FILE_NAME_LEN] = {0}; char localDir[VMDIR_MAX_FILE_NAME_LEN] = {0}; + char localXlogDir[VMDIR_MAX_FILE_NAME_LEN] = {0}; char localFilename[VMDIR_MAX_FILE_NAME_LEN] = {0}; PSTR pszDcAccountPwd = NULL; PVMDIR_SERVER_CONTEXT hServer = NULL; DWORD low_xlognum = 0; + DWORD high_xlognum = 0; DWORD xlognum = 0; DWORD remoteDbSizeMb = 0; DWORD remoteDbMapSizeMb = 0; PBYTE pDbPath = NULL; + BOOLEAN bMdbWalEnable = FALSE; + #ifndef _WIN32 const char fileSeperator = '/'; #else @@ -183,10 +191,21 @@ _VmDirGetRemoteDBUsingRPC( retVal, pszHostname ); VMDIR_LOG_INFO( VMDIR_LOG_MASK_ALL, "_VmDirGetRemoteDBUsingRPC: Connected to the replication partner (%s).", pszHostname ); - //Set backend to read-only mode - currently no backend is supporting WAL yet. - retVal = VmDirSetBackendState (hServer, 1, &low_xlognum, &remoteDbSizeMb, &remoteDbMapSizeMb, pDbPath, VMDIR_MAX_FILE_NAME_LEN); - BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), - "_VmDirGetRemoteDBUsingRPC: VmDirSetBackendState failed with error: %d", retVal ); + VmDirGetMdbWalEnable(&bMdbWalEnable); + + if (bMdbWalEnable) + { + //Set remote server backend to KEEPXLOGS mode + retVal = VmDirSetBackendState (hServer, MDB_STATE_KEEPXLOGS, &low_xlognum, &remoteDbSizeMb, + &remoteDbMapSizeMb, pDbPath, VMDIR_MAX_FILE_NAME_LEN); + } else + { + //Set remote server backend to ReadOnly mode + retVal = VmDirSetBackendState (hServer, MDB_STATE_READONLY, &low_xlognum, &remoteDbSizeMb, + &remoteDbMapSizeMb, pDbPath, VMDIR_MAX_FILE_NAME_LEN); + } + BAIL_ON_VMDIR_ERROR_WITH_MSG(retVal, (pszLocalErrorMsg), + "_VmDirGetRemoteDBUsingRPC: VmDirSetBackendState failed, WalEnabled: %d, error: %d", bMdbWalEnable, retVal); retVal = VmDirStringPrintFA( localDir, VMDIR_MAX_FILE_NAME_LEN, "%s%c%s", dbHomeDir, fileSeperator, LOCAL_PARTNER_DIR); BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), @@ -195,6 +214,17 @@ _VmDirGetRemoteDBUsingRPC( retVal = _VmDirMkdir(localDir, 0700); BAIL_ON_VMDIR_ERROR( retVal ); + if (low_xlognum > 0) + { + retVal = VmDirStringPrintFA( localXlogDir, VMDIR_MAX_FILE_NAME_LEN, "%s%c%s", localDir, fileSeperator, VMDIR_MDB_XLOGS_DIR_NAME); + BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), + "_VmDirGetRemoteDBUsingRPC: VmDirStringPrintFA() call failed with error: %d", retVal ); + + retVal = _VmDirMkdir(localXlogDir, 0700); + BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), + "_VmDirGetRemoteDBUsingRPC: _VmDirMkdir() call failed with error: %d %s", retVal ); + } + retVal = VmDirStringPrintFA( dbRemoteFilename, VMDIR_MAX_FILE_NAME_LEN, "%s/%s", (char *)pDbPath, VMDIR_MDB_DATA_FILE_NAME ); @@ -213,16 +243,48 @@ _VmDirGetRemoteDBUsingRPC( retVal = _VmDirGetRemoteDBFileUsingRPC( hServer, dbRemoteFilename, localFilename, remoteDbSizeMb, remoteDbMapSizeMb ); BAIL_ON_VMDIR_ERROR( retVal ); + if (low_xlognum == 0) + { + VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, + "_VmDirGetRemoteDBUsingRPC: complete MDB cold copy - WAL not supported by remote"); + goto cleanup; + } + + //Query current xlog number + retVal = VmDirSetBackendState (hServer, MDB_STATE_GETXLOGNUM, &high_xlognum, &remoteDbSizeMb, &remoteDbMapSizeMb, pDbPath, VMDIR_MAX_FILE_NAME_LEN); + BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), + "_VmDirGetRemoteDBUsingRPC: VmDirSetBackendState failed to get current xlog: %d", retVal ); + + VMDIR_LOG_INFO( VMDIR_LOG_MASK_ALL, "_VmDirGetRemoteDBUsingRPC: start transfering XLOGS from %d to %d", low_xlognum, high_xlognum); + for (xlognum = low_xlognum; xlognum <= high_xlognum; xlognum++) + { + retVal = VmDirStringPrintFA( dbRemoteFilename, VMDIR_MAX_FILE_NAME_LEN, "%s%c%s%c%lu", dbHomeDir, fileSeperator, + VMDIR_MDB_XLOGS_DIR_NAME, fileSeperator, xlognum ); + BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), + "_VmDirGetRemoteDBUsingRPC: VmDirStringPrintFA() call failed with error: %d", retVal ); + + retVal = VmDirStringPrintFA( localFilename, VMDIR_MAX_FILE_NAME_LEN, "%s%c%lu", localXlogDir, fileSeperator, xlognum); + BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), + "_VmDirGetRemoteDBUsingRPC: VmDirStringPrintFA() call failed with error: %d", retVal ); + + retVal = _VmDirGetRemoteDBFileUsingRPC( hServer, dbRemoteFilename, localFilename, 0, 0); + BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), + "_VmDirGetRemoteDBUsingRPC: _VmDirGetRemoteDBFileUsingRPC() call failed with error: %d", retVal ); + } + + VMDIR_LOG_INFO( VMDIR_LOG_MASK_ALL, "_VmDirGetRemoteDBUsingRPC: complete transfering XLOGS from %d to %d", low_xlognum, high_xlognum); + cleanup: if (hServer) { - //clear backend read-only mode - VmDirSetBackendState (hServer, 0, &xlognum, &remoteDbSizeMb, &remoteDbMapSizeMb, pDbPath, VMDIR_MAX_FILE_NAME_LEN); + //clear backend transfering xlog files mode. + VmDirSetBackendState (hServer, MDB_STATE_CLEAR, &xlognum, &remoteDbSizeMb, &remoteDbMapSizeMb, pDbPath, VMDIR_MAX_FILE_NAME_LEN); VmDirCloseServer( hServer); } VMDIR_SAFE_FREE_MEMORY(pszLocalErrorMsg); VMDIR_SAFE_FREE_MEMORY(pDbPath); VMDIR_SECURE_FREE_STRINGA(pszDcAccountPwd); + *pbHasXlog = (low_xlognum > 0); return retVal; error: @@ -230,7 +292,6 @@ _VmDirGetRemoteDBUsingRPC( VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, "%s", VDIR_SAFE_STRING(pszLocalErrorMsg) ); goto cleanup; } - DWORD VmDirReadDatabaseFile( PVMDIR_SERVER_CONTEXT hServer, @@ -254,7 +315,7 @@ _VmDirGetRemoteDBFileUsingRPC( UINT32 remoteDbMapSizeMb) { //read block size of one MB. -#define VMDIR_DB_READ_BLOCK_SIZE (1<<20) +#define VMDIR_DB_READ_BLOCK_SIZE (1<<23) DWORD retVal = 0; #ifdef _WIN32 @@ -391,15 +452,16 @@ _VmDirGetRemoteDBFileUsingRPC( static int _VmDirSwapDB( - PCSTR dbHomeDir) + PCSTR dbHomeDir, + BOOLEAN bHasXlog) { int retVal = LDAP_SUCCESS; - char dbExistingFilename[VMDIR_MAX_FILE_NAME_LEN] = {0}; - char dbNewFilename[VMDIR_MAX_FILE_NAME_LEN] = {0}; - PVDIR_BACKEND_INTERFACE pBE = NULL; + char dbExistingName[VMDIR_MAX_FILE_NAME_LEN] = {0}; + char dbNewName[VMDIR_MAX_FILE_NAME_LEN] = {0}; PSTR pszLocalErrorMsg = NULL; int errorCode = 0; BOOLEAN bLegacyDataLoaded = FALSE; + PVDIR_BACKEND_INTERFACE pBE = NULL; #ifndef _WIN32 const char fileSeperator = '/'; @@ -421,45 +483,82 @@ _VmDirSwapDB( VmDirBackendContentFree(pBE); // move .mdb files - retVal = VmDirStringPrintFA( dbExistingFilename, VMDIR_MAX_FILE_NAME_LEN, "%s%c%s%c%s", dbHomeDir, fileSeperator, + retVal = VmDirStringPrintFA( dbExistingName, VMDIR_MAX_FILE_NAME_LEN, "%s%c%s%c%s", dbHomeDir, fileSeperator, LOCAL_PARTNER_DIR, fileSeperator, VMDIR_MDB_DATA_FILE_NAME); BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), "_VmDirSwapDB: VmDirStringPrintFA() call failed with error: %d", retVal ); - retVal = VmDirStringPrintFA( dbNewFilename, VMDIR_MAX_FILE_NAME_LEN, "%s%c%s", dbHomeDir, fileSeperator, + retVal = VmDirStringPrintFA( dbNewName, VMDIR_MAX_FILE_NAME_LEN, "%s%c%s", dbHomeDir, fileSeperator, VMDIR_MDB_DATA_FILE_NAME ); BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), "_VmDirSwapDB: VmDirStringPrintFA() call failed with error: %d", retVal ); #ifdef WIN32 - if (MoveFileEx(dbExistingFilename, dbNewFilename, MOVEFILE_COPY_ALLOWED|MOVEFILE_REPLACE_EXISTING) == 0) + if (MoveFileEx(dbExistingName, dbNewName, MOVEFILE_COPY_ALLOWED|MOVEFILE_REPLACE_EXISTING) == 0) { retVal = LDAP_OPERATIONS_ERROR; errorCode = GetLastError(); #else - if (rename(dbExistingFilename, dbNewFilename) != 0) + if (rename(dbExistingName, dbNewName) != 0) { retVal = LDAP_OPERATIONS_ERROR; errorCode = errno; #endif BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), - "_VmDirSwapDB: rename file from %s to %s failed, errno %d", dbExistingFilename, dbNewFilename, errorCode ); + "_VmDirSwapDB: rename file from %s to %s failed, errno %d", dbExistingName, dbNewName, errorCode ); } - retVal = VmDirStringPrintFA( dbExistingFilename, VMDIR_MAX_FILE_NAME_LEN, "%s%c%s", dbHomeDir, fileSeperator, LOCAL_PARTNER_DIR); + retVal = VmDirStringPrintFA(dbNewName, VMDIR_MAX_FILE_NAME_LEN, "%s%c%s%c%s", dbHomeDir, fileSeperator, VMDIR_MDB_XLOGS_DIR_NAME); + BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), + "_VmDirSwapDB: VmDirStringPrintFA() call failed with error: %d", retVal ); + + if (bHasXlog) + { + //move xlog directory + retVal = VmDirStringPrintFA(dbExistingName, VMDIR_MAX_FILE_NAME_LEN, "%s%c%s%c%s", dbHomeDir, fileSeperator, + LOCAL_PARTNER_DIR, fileSeperator, VMDIR_MDB_XLOGS_DIR_NAME); + BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), + "_VmDirSwapDB: VmDirStringPrintFA() call failed with error: %d", retVal ); + +#ifdef WIN32 + if (MoveFileEx(dbExistingName, dbNewName, MOVEFILE_COPY_ALLOWED|MOVEFILE_REPLACE_EXISTING) == 0) + { + retVal = LDAP_OPERATIONS_ERROR; + errorCode = GetLastError(); +#else + if (rmdir(dbNewName) != 0) + { + retVal = LDAP_OPERATIONS_ERROR; + errorCode = errno; + BAIL_ON_VMDIR_ERROR_WITH_MSG(retVal, (pszLocalErrorMsg), "_VmDirSwapDB cannot remove directory %s, errno %d", + dbNewName, errorCode); + } + + if (rename(dbExistingName, dbNewName) != 0) + { + retVal = LDAP_OPERATIONS_ERROR; + errorCode = errno; +#endif + BAIL_ON_VMDIR_ERROR_WITH_MSG(retVal, (pszLocalErrorMsg), "_VmDirSwapDB cannot move directory from %s to %s, errno %d", + dbNewName, dbExistingName, errorCode); + } + } + + retVal = VmDirStringPrintFA(dbExistingName, VMDIR_MAX_FILE_NAME_LEN, "%s%c%s", dbHomeDir, fileSeperator, LOCAL_PARTNER_DIR); BAIL_ON_VMDIR_ERROR_WITH_MSG( retVal, (pszLocalErrorMsg), "_VmDirSwapDB: VmDirStringPrintFA() call failed with error: %d", retVal ); #ifdef WIN32 - if (RemoveDirectory(dbExistingFilename)==0) + if (RemoveDirectory(dbExistingName)==0) { errorCode = GetLastError(); #else - if (rmdir(dbExistingFilename) != 0) + if (rmdir(dbExistingName)) { errorCode = errno; #endif - VMDIR_LOG_WARNING(VMDIR_LOG_MASK_ALL, "_VmDirSwapDB cannot remove directory %s, errno %d", dbExistingFilename, errorCode); + + VMDIR_LOG_WARNING(VMDIR_LOG_MASK_ALL, "cannot remove directory %s errno %d", dbExistingName, errorCode); } VmDirdStateSet(VMDIRD_STATE_STARTUP); diff --git a/vmdir/server/replication/replentry.c b/vmdir/server/replication/replentry.c index 9cdb36c2f..b0e2811d7 100644 --- a/vmdir/server/replication/replentry.c +++ b/vmdir/server/replication/replentry.c @@ -119,6 +119,39 @@ _VmDirIsBenignReplConflict( PVDIR_ENTRY pConsumerEntry ); +static +VOID +_VmDirLogReplAddEntryContent( + PVMDIR_REPLICATION_PAGE_ENTRY pPageEntry, + PVDIR_ENTRY pEntry + ); + +static +VOID +_VmDirLogReplModifyEntryContent( + PVMDIR_REPLICATION_PAGE_ENTRY pPageEntry, + PVDIR_ENTRY pEntry + ); + +static +VOID +_VmDirLogReplDeleteEntryContent( + PVMDIR_REPLICATION_PAGE_ENTRY pPageEntry, + PVDIR_ENTRY pEntry + ); + +static +VOID +_VmDirLogReplEntryContent( + PVDIR_ENTRY pEntry + ); + +static +VOID +_VmDirLogReplModifyModContent( + ModifyReq* pModReq + ); + // Replicate Add Entry operation int @@ -155,6 +188,8 @@ ReplAddEntry( retVal = VmDirParseEntry( &op ); BAIL_ON_VMDIR_ERROR( retVal ); + _VmDirLogReplAddEntryContent(pPageEntry, op.request.addReq.pEntry); + op.pBEIF = VmDirBackendSelect(pEntry->dn.lberbv.bv_val); assert(op.pBEIF); @@ -328,6 +363,8 @@ ReplDeleteEntry( retVal = ReplFixUpEntryDn(tmpAddOp.request.addReq.pEntry); BAIL_ON_VMDIR_ERROR( retVal ); + _VmDirLogReplDeleteEntryContent(pPageEntry, tmpAddOp.request.addReq.pEntry); + if (VmDirBervalContentDup( &tmpAddOp.reqDn, &mr->dn ) != 0) { VMDIR_LOG_ERROR(VMDIR_LOG_MASK_ALL, "ReplDeleteEntry: BervalContentDup failed." ); @@ -436,6 +473,8 @@ ReplModifyEntry( retVal = ReplFixUpEntryDn(&e); BAIL_ON_VMDIR_ERROR( retVal ); + _VmDirLogReplModifyEntryContent(pPageEntry, &e); + if (VmDirBervalContentDup( &e.dn, &mr->dn ) != 0) { VMDIR_LOG_ERROR(VMDIR_LOG_MASK_ALL, "SetupReplModifyRequest: BervalContentDup failed." ); @@ -551,6 +590,8 @@ ReplModifyEntry( modOp.ulPartnerUSN = pPageEntry->ulPartnerUSN; + _VmDirLogReplModifyModContent(&modOp.request.modifyReq); + // SJ-TBD: What happens when DN of the entry has changed in the meanwhile? => conflict resolution. // Should objectGuid, instead of DN, be used to uniquely identify an object? if ((retVal = VmDirInternalModifyEntry( &modOp )) != LDAP_SUCCESS) @@ -581,6 +622,8 @@ ReplModifyEntry( mr = &(modOp.request.modifyReq); if (mr->mods != NULL) { + _VmDirLogReplModifyModContent(&modOp.request.modifyReq); + if ((retVal = VmDirInternalModifyEntry( &modOp )) != LDAP_SUCCESS) { retVal = modOp.ldapResult.errCode; @@ -1901,3 +1944,131 @@ _VmDirAttachValueMetaData( VmDirFreeBerval(pAVmeta); goto cleanup; } + + +static +VOID +_VmDirLogReplEntryContent( + PVDIR_ENTRY pEntry + ) +{ + PVDIR_ATTRIBUTE pAttr = NULL; + int iCnt = 0; + + for (pAttr = pEntry->attrs; pAttr; pAttr = pAttr->next) + { + for (iCnt=0; iCnt < pAttr->numVals; iCnt++) + { + PCSTR pszLogValue = (0 == VmDirStringCompareA( pAttr->type.lberbv.bv_val, ATTR_USER_PASSWORD, FALSE)) ? + "XXX" : pAttr->vals[iCnt].lberbv_val; + + if (iCnt < MAX_NUM_CONTENT_LOG) + { + VMDIR_LOG_INFO( LDAP_DEBUG_REPL_ATTR, "%s %s %d (%.*s)", + __FUNCTION__, + pAttr->type.lberbv.bv_val, + iCnt+1, + VMDIR_MIN(pAttr->vals[iCnt].lberbv_len, VMDIR_MAX_LOG_OUTPUT_LEN), + VDIR_SAFE_STRING(pszLogValue)); + } + else if (iCnt == MAX_NUM_CONTENT_LOG) + { + VMDIR_LOG_INFO( LDAP_DEBUG_REPL_ATTR, "%s Total value count %d)", __FUNCTION__, pAttr->numVals); + } + else + { + break; + } + } + } +} + +static +VOID +_VmDirLogReplAddEntryContent( + PVMDIR_REPLICATION_PAGE_ENTRY pPageEntry, + PVDIR_ENTRY pEntry + ) +{ + VMDIR_LOG_INFO( LDAP_DEBUG_REPL_ATTR, + "%s, DN:%s, SYNC_STATE:ADD, partner USN:%" PRId64, + __FUNCTION__, + pPageEntry->pszDn, + pPageEntry->ulPartnerUSN); + + if (VmDirLogLevelAndMaskTest(VMDIR_LOG_VERBOSE, LDAP_DEBUG_REPL_ATTR)) + { + _VmDirLogReplEntryContent(pEntry); + } +} + +static +VOID +_VmDirLogReplDeleteEntryContent( + PVMDIR_REPLICATION_PAGE_ENTRY pPageEntry, + PVDIR_ENTRY pEntry + ) +{ + VMDIR_LOG_INFO( LDAP_DEBUG_REPL_ATTR, + "%s, DN:%s SYNC_STATE:Delete, partner USN:%" PRId64 " local DN: %s", + __FUNCTION__, + pPageEntry->pszDn, + pPageEntry->ulPartnerUSN, + pEntry->dn.lberbv_val); +} + +static +VOID +_VmDirLogReplModifyEntryContent( + PVMDIR_REPLICATION_PAGE_ENTRY pPageEntry, + PVDIR_ENTRY pEntry + ) +{ + VMDIR_LOG_INFO( LDAP_DEBUG_REPL_ATTR, + "%s, DN:%s, SYNC_STATE:Modify, partner USN:%" PRId64, + __FUNCTION__, + pPageEntry->pszDn, + pPageEntry->ulPartnerUSN); + + _VmDirLogReplEntryContent(pEntry); +} + +static +VOID +_VmDirLogReplModifyModContent( + ModifyReq* pModReq + ) +{ + PVDIR_MODIFICATION pMod = pModReq->mods; + int iCnt = 0; + + for (; pMod; pMod = pMod->next) + { + for (iCnt=0; iCnt < pMod->attr.numVals; iCnt++) + { + PCSTR pszLogValue = (0 == VmDirStringCompareA( pMod->attr.type.lberbv.bv_val, ATTR_USER_PASSWORD, FALSE)) ? + "XXX" : pMod->attr.vals[iCnt].lberbv_val; + + if (iCnt < MAX_NUM_CONTENT_LOG) + { + VMDIR_LOG_INFO( VMDIR_LOG_MASK_ALL, + "%s MOD %d, %s, %s: (%.*s)", + __FUNCTION__, + iCnt+1, + VmDirLdapModOpTypeToName(pMod->operation), + VDIR_SAFE_STRING(pMod->attr.type.lberbv.bv_val), + VMDIR_MIN(pMod->attr.vals[iCnt].lberbv_len, VMDIR_MAX_LOG_OUTPUT_LEN), + VDIR_SAFE_STRING(pszLogValue)); + } + else if (iCnt == MAX_NUM_CONTENT_LOG) + { + VMDIR_LOG_INFO( VMDIR_LOG_MASK_ALL, + "%s Total value count %d)", __FUNCTION__, pMod->attr.numVals); + } + else + { + break; + } + } + } +} diff --git a/vmdir/server/rest-head/auth.c b/vmdir/server/rest-head/auth.c index 666775a8c..c67127a71 100644 --- a/vmdir/server/rest-head/auth.c +++ b/vmdir/server/rest-head/auth.c @@ -110,6 +110,11 @@ VmDirRESTAuthViaBasic( pszPasswd++; dwError = VmDirUPNToDN(pszDecode, &pszBindDN); + // we want this error to be mapped to invalid credentials + if (dwError == VMDIR_ERROR_ENTRY_NOT_FOUND) + { + dwError = VMDIR_ERROR_AUTH_BAD_DATA; + } BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirExternalOperationCreate( diff --git a/vmdir/server/rest-head/defines.h b/vmdir/server/rest-head/defines.h index ef1220296..10d9ee9ea 100644 --- a/vmdir/server/rest-head/defines.h +++ b/vmdir/server/rest-head/defines.h @@ -16,11 +16,17 @@ #define REST_API_SPEC VMDIR_CONFIG_DIR VMDIR_PATH_SEPARATOR_STR "vmdir-rest.json" #define VMDIR_HTTP_DEBUGLOGFILE VMDIR_LOG_DIR VMDIR_PATH_SEPARATOR_STR "vmdir-rest-HTTP.log" #define VMDIR_HTTPS_DEBUGLOGFILE VMDIR_LOG_DIR VMDIR_PATH_SEPARATOR_STR "vmdir-rest-HTTPS.log" -#define VMDIR_REST_CLIENTCNT "64" -#define VMDIR_REST_WORKERTHCNT "64" +#define VMDIR_REST_CLIENTCNT 64 +#define VMDIR_REST_WORKERTHCNT 64 #define MAX_REST_PAYLOAD_LENGTH 4096 +#define VMDIR_REST_CONN_TIMEOUT_SEC 30 +#define VMDIR_MAX_DATA_PER_CONN_MB 25 +#define VMDIR_HTTP_DAEMON_NAME "vmdird-http"; +#define VMDIR_HTTPS_DAEMON_NAME "vmdird-https"; +#define VMDIR_REST_STOP_TIMEOUT_SEC 10 + // OIDC #define VMDIR_REST_OIDC_SERVER "localhost" #define VMDIR_REST_OIDC_PORT 443 diff --git a/vmdir/server/rest-head/ldapapi.c b/vmdir/server/rest-head/ldapapi.c index 36fcbea2b..c510c1afd 100644 --- a/vmdir/server/rest-head/ldapapi.c +++ b/vmdir/server/rest-head/ldapapi.c @@ -394,13 +394,16 @@ VmDirRESTLdapGetHttpError( case LDAP_TYPE_OR_VALUE_EXISTS: case LDAP_OBJECT_CLASS_VIOLATION: case LDAP_ALREADY_EXISTS: - case LDAP_NO_SUCH_OBJECT: case LDAP_CONSTRAINT_VIOLATION: case LDAP_NOT_ALLOWED_ON_NONLEAF: case LDAP_PROTOCOL_ERROR: httpStatus = HTTP_BAD_REQUEST; break; + case LDAP_NO_SUCH_OBJECT: + httpStatus = HTTP_NOT_FOUND; + break; + case LDAP_INVALID_CREDENTIALS: case LDAP_INSUFFICIENT_ACCESS: case LDAP_AUTH_METHOD_NOT_SUPPORTED: diff --git a/vmdir/server/rest-head/libmain.c b/vmdir/server/rest-head/libmain.c index c7d690d91..89e73ab46 100644 --- a/vmdir/server/rest-head/libmain.c +++ b/vmdir/server/rest-head/libmain.c @@ -49,6 +49,12 @@ _VmDirRESTServerShutdownHTTPS( VOID ); +static +VOID +_VmDirFreeRESTHandle( + PVMREST_HANDLE pHandle + ); + DWORD VmDirRESTServerInit( VOID @@ -94,6 +100,7 @@ VmDirRESTServerInit( return dwError; error: + VmDirRESTServerShutdown(); VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, "%s failed, error (%d)", @@ -122,14 +129,14 @@ _VmDirRESTServerInitHTTP( { DWORD dwError = 0; REST_CONF config = {0}; - PREST_PROCESSOR pHandlers = &sVmDirRESTHandlers; - PREST_API_MODULE pModule = NULL; + PREST_PROCESSOR pHandlers = &sVmDirRESTHandlers; + PREST_API_MODULE pModule = NULL; + PVMREST_HANDLE pHTTPHandle = NULL; /* - * pszHTTPListenPort can never be NULL because of default values assigned to them - * if Port string is empty, it means user wants to disable corresponding service + * dwHTTPListenPort is '0' then user wants to disable HTTP endpoint */ - if (IsNullOrEmptyString(gVmdirGlobals.pszHTTPListenPort)) + if (gVmdirGlobals.dwHTTPListenPort == 0) { VMDIR_LOG_WARNING( VMDIR_LOG_MASK_ALL, @@ -138,14 +145,23 @@ _VmDirRESTServerInitHTTP( goto cleanup; } - config.pSSLCertificate = RSA_SERVER_CERT; - config.pSSLKey = RSA_SERVER_KEY; - config.pServerPort = gVmdirGlobals.pszHTTPListenPort; - config.pDebugLogFile = VMDIR_HTTP_DEBUGLOGFILE; - config.pClientCount = VMDIR_REST_CLIENTCNT; - config.pMaxWorkerThread = VMDIR_REST_WORKERTHCNT; - - dwError = VmRESTInit(&config, NULL, &gpVdirRestHTTPHandle); + config.serverPort = gVmdirGlobals.dwHTTPListenPort; + config.connTimeoutSec = VMDIR_REST_CONN_TIMEOUT_SEC; + config.maxDataPerConnMB = VMDIR_MAX_DATA_PER_CONN_MB; + config.pSSLContext = NULL; + config.nWorkerThr = VMDIR_REST_WORKERTHCNT; + config.nClientCnt = VMDIR_REST_CLIENTCNT; + config.SSLCtxOptionsFlag = 0; + config.pszSSLCertificate = NULL; + config.pszSSLKey = NULL; + config.pszSSLCipherList = NULL; + config.pszDebugLogFile = VMDIR_HTTP_DEBUGLOGFILE; + config.pszDaemonName = VMDIR_HTTP_DAEMON_NAME; + config.isSecure = FALSE; + config.useSysLog = TRUE; + config.debugLogLevel = VMREST_LOG_LEVEL_INFO; + + dwError = VmRESTInit(&config, &pHTTPHandle); BAIL_ON_VMDIR_ERROR(dwError); for (pModule = gpVdirRestApiDef->pModules; pModule; pModule = pModule->pNext) @@ -154,18 +170,21 @@ _VmDirRESTServerInitHTTP( for (; pEndPoint; pEndPoint = pEndPoint->pNext) { dwError = VmRESTRegisterHandler( - gpVdirRestHTTPHandle, pEndPoint->pszName, pHandlers, NULL); + pHTTPHandle, pEndPoint->pszName, pHandlers, NULL); BAIL_ON_VMDIR_ERROR(dwError); } } - dwError = VmRESTStart(gpVdirRestHTTPHandle); + dwError = VmRESTStart(pHTTPHandle); BAIL_ON_VMDIR_ERROR(dwError); + gpVdirRestHTTPHandle = pHTTPHandle; + cleanup: return dwError; error: + _VmDirFreeRESTHandle(pHTTPHandle); VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, "%s failed, error (%d)", @@ -182,17 +201,17 @@ _VmDirRESTServerInitHTTPS( ) { DWORD dwError = 0; - PSTR pszCert = NULL; - PSTR pszKey = NULL; REST_CONF config = {0}; PREST_PROCESSOR pHandlers = &sVmDirRESTHandlers; PREST_API_MODULE pModule = NULL; + PVMREST_HANDLE pHTTPSHandle = NULL; /* - * pszHTTPSListenPort can never be NULL because of default values assigned to them - * if Port string is empty, it means user wants to disable corresponding service + * dwHTTPSListenPort is '0' then user wants to disable HTTPS endpoint + * Initializing openssl context is treated as soft fail, gpVdirSslCtx can be NULL + * If gpVdirSslCtx NULL, don't start the service */ - if (IsNullOrEmptyString(gVmdirGlobals.pszHTTPSListenPort)) + if (gVmdirGlobals.dwHTTPSListenPort == 0 || gVmdirGlobals.gpVdirSslCtx == NULL) { VMDIR_LOG_WARNING( VMDIR_LOG_MASK_ALL, @@ -201,24 +220,23 @@ _VmDirRESTServerInitHTTPS( goto cleanup; } - config.pSSLCertificate = NULL; - config.pSSLKey = NULL; - config.pServerPort = gVmdirGlobals.pszHTTPSListenPort; - config.pDebugLogFile = VMDIR_HTTPS_DEBUGLOGFILE; - config.pClientCount = VMDIR_REST_CLIENTCNT; - config.pMaxWorkerThread = VMDIR_REST_WORKERTHCNT; - - dwError = VmRESTInit(&config, NULL, &gpVdirRestHTTPSHandle); - BAIL_ON_VMDIR_ERROR(dwError); - - //Get Certificate and Key from VECS and Set it to Rest Engine - dwError = VmDirGetVecsMachineCert(&pszCert, &pszKey); - BAIL_ON_VMDIR_ERROR(dwError); - - dwError = VmRESTSetSSLInfo(gpVdirRestHTTPSHandle, pszCert, VmDirStringLenA(pszCert)+1, SSL_DATA_TYPE_CERT); - BAIL_ON_VMDIR_ERROR(dwError); - - dwError = VmRESTSetSSLInfo(gpVdirRestHTTPSHandle, pszKey, VmDirStringLenA(pszKey)+1, SSL_DATA_TYPE_KEY); + config.serverPort = gVmdirGlobals.dwHTTPSListenPort; + config.connTimeoutSec = VMDIR_REST_CONN_TIMEOUT_SEC; + config.maxDataPerConnMB = VMDIR_MAX_DATA_PER_CONN_MB; + config.pSSLContext = gVmdirGlobals.gpVdirSslCtx; + config.nWorkerThr = VMDIR_REST_WORKERTHCNT; + config.nClientCnt = VMDIR_REST_CLIENTCNT; + config.SSLCtxOptionsFlag = 0; + config.pszSSLCertificate = NULL; + config.pszSSLKey = NULL; + config.pszSSLCipherList = NULL; + config.pszDebugLogFile = VMDIR_HTTPS_DEBUGLOGFILE; + config.pszDaemonName = VMDIR_HTTPS_DAEMON_NAME; + config.isSecure = TRUE; + config.useSysLog = TRUE; + config.debugLogLevel = VMREST_LOG_LEVEL_INFO; + + dwError = VmRESTInit(&config, &pHTTPSHandle); BAIL_ON_VMDIR_ERROR(dwError); for (pModule = gpVdirRestApiDef->pModules; pModule; pModule = pModule->pNext) @@ -227,20 +245,21 @@ _VmDirRESTServerInitHTTPS( for (; pEndPoint; pEndPoint = pEndPoint->pNext) { dwError = VmRESTRegisterHandler( - gpVdirRestHTTPSHandle, pEndPoint->pszName, pHandlers, NULL); + pHTTPSHandle, pEndPoint->pszName, pHandlers, NULL); BAIL_ON_VMDIR_ERROR(dwError); } } - dwError = VmRESTStart(gpVdirRestHTTPSHandle); + dwError = VmRESTStart(pHTTPSHandle); BAIL_ON_VMDIR_ERROR(dwError); + gpVdirRestHTTPSHandle = pHTTPSHandle; + cleanup: - VMDIR_SAFE_FREE_MEMORY(pszCert); - VMDIR_SAFE_FREE_MEMORY(pszKey); return dwError; error: + _VmDirFreeRESTHandle(pHTTPSHandle); VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, "%s failed, error (%d)", @@ -256,32 +275,8 @@ _VmDirRESTServerShutdownHTTP( VOID ) { - PREST_API_MODULE pModule = NULL; - - if (IsNullOrEmptyString(gVmdirGlobals.pszHTTPListenPort)) - { - //No operation - HTTP port was not initialized - return; - } - - if (gpVdirRestHTTPHandle) - { - VmRESTStop(gpVdirRestHTTPHandle); - if (gpVdirRestApiDef) - { - pModule = gpVdirRestApiDef->pModules; - for (; pModule; pModule = pModule->pNext) - { - PREST_API_ENDPOINT pEndPoint = pModule->pEndPoints; - for (; pEndPoint; pEndPoint = pEndPoint->pNext) - { - (VOID)VmRESTUnRegisterHandler( - gpVdirRestHTTPHandle, pEndPoint->pszName); - } - } - } - VmRESTShutdown(gpVdirRestHTTPHandle); - } + _VmDirFreeRESTHandle(gpVdirRestHTTPHandle); + gpVdirRestHTTPHandle = NULL; } static @@ -290,17 +285,38 @@ _VmDirRESTServerShutdownHTTPS( VOID ) { - PREST_API_MODULE pModule = NULL; + _VmDirFreeRESTHandle(gpVdirRestHTTPSHandle); + gpVdirRestHTTPSHandle = NULL; +} - if (IsNullOrEmptyString(gVmdirGlobals.pszHTTPSListenPort)) - { - //No operation - HTTPS port was not initialized - return; - } +static +VOID +_VmDirFreeRESTHandle( + PVMREST_HANDLE pHandle + ) +{ + DWORD dwError = 0; + PREST_API_MODULE pModule = NULL; - if (gpVdirRestHTTPSHandle) + if (pHandle) { - VmRESTStop(gpVdirRestHTTPSHandle); + /* + * REST library have detached threads, maximum time out specified is the max time + * allowed for the threads to finish their execution. + * If finished early, it will return success. + * If not able to finish in specified time, failure will be returned + */ + dwError = VmRESTStop(pHandle, VMDIR_REST_STOP_TIMEOUT_SEC); + + if (dwError != 0) + { + VMDIR_LOG_WARNING( + VMDIR_LOG_MASK_ALL, + "%s: Rest stop error: %d", + __FUNCTION__, + dwError); + } + if (gpVdirRestApiDef) { pModule = gpVdirRestApiDef->pModules; @@ -310,11 +326,11 @@ _VmDirRESTServerShutdownHTTPS( for (; pEndPoint; pEndPoint = pEndPoint->pNext) { (VOID)VmRESTUnRegisterHandler( - gpVdirRestHTTPSHandle, pEndPoint->pszName); + pHandle, pEndPoint->pszName); } } } - VmRESTShutdown(gpVdirRestHTTPSHandle); + VmRESTShutdown(pHandle); } } diff --git a/vmdir/server/rest-head/operation.c b/vmdir/server/rest-head/operation.c index 8d4287102..c17b24f75 100644 --- a/vmdir/server/rest-head/operation.c +++ b/vmdir/server/rest-head/operation.c @@ -91,8 +91,8 @@ VmDirRESTOperationReadRequest( dwError = VmRESTGetHttpMethod(pRestReq, &pRestOp->pszMethod); BAIL_ON_VMDIR_ERROR(dwError); - // read request URI - dwError = VmRESTGetHttpURI(pRestReq, &pRestOp->pszPath); + // read request URI. TRUE - req c-rest-engine to decode URI + dwError = VmRESTGetHttpURI(pRestReq, TRUE, &pRestOp->pszPath); BAIL_ON_VMDIR_ERROR(dwError); pszTmp = VmDirStringChrA(pRestOp->pszPath, '?'); diff --git a/vmdir/server/vmdir/defines.h b/vmdir/server/vmdir/defines.h index 53a495abb..d2fcf1006 100644 --- a/vmdir/server/vmdir/defines.h +++ b/vmdir/server/vmdir/defines.h @@ -212,25 +212,25 @@ /*.pszValue = */ NULL \ }, \ { \ - /*.pszName = */ VMDIR_REG_KEY_HTTP_LISTEN_PORT, \ - /*.Type = */ VMDIR_CONFIG_VALUE_TYPE_STRING, \ - /*.RegDataType = */ REG_SZ, \ + /*.pszName = */ VMDIR_REG_KEY_HTTP_LISTEN_PORT, \ + /*.Type = */ VMDIR_CONFIG_VALUE_TYPE_DWORD, \ + /*.RegDataType = */ REG_DWORD, \ /*.dwMin = */ 0, \ - /*.dwMax = */ 0, \ - /*.dwDefault = */ 0, \ + /*.dwMax = */ 65535, \ + /*.dwDefault = */ DEFAULT_HTTP_PORT_NUM, \ /*.dwValue = */ 0, \ - /*.pszDefault = */ DEFAULT_HTTP_PORT_STR, \ + /*.pszDefault = */ NULL, \ /*.pszValue = */ NULL \ }, \ { \ /*.pszName = */ VMDIR_REG_KEY_HTTPS_LISTEN_PORT, \ - /*.Type = */ VMDIR_CONFIG_VALUE_TYPE_STRING, \ - /*.RegDataType = */ REG_SZ, \ + /*.Type = */ VMDIR_CONFIG_VALUE_TYPE_DWORD, \ + /*.RegDataType = */ REG_DWORD, \ /*.dwMin = */ 0, \ - /*.dwMax = */ 0, \ - /*.dwDefault = */ 0, \ + /*.dwMax = */ 65535, \ + /*.dwDefault = */ DEFAULT_HTTPS_PORT_NUM, \ /*.dwValue = */ 0, \ - /*.pszDefault = */ DEFAULT_HTTPS_PORT_STR, \ + /*.pszDefault = */ NULL, \ /*.pszValue = */ NULL \ }, \ { \ diff --git a/vmdir/server/vmdir/globals.c b/vmdir/server/vmdir/globals.c index e5c9a3f6a..3e99f8769 100644 --- a/vmdir/server/vmdir/globals.c +++ b/vmdir/server/vmdir/globals.c @@ -51,8 +51,8 @@ VMDIR_GLOBALS gVmdirGlobals = VMDIR_SF_INIT(.dwLdapConnectPorts, 0), VMDIR_SF_INIT(.pdwLdapsConnectPorts, NULL), VMDIR_SF_INIT(.dwLdapsConnectPorts, 0), - VMDIR_SF_INIT(.pszHTTPListenPort, NULL), - VMDIR_SF_INIT(.pszHTTPSListenPort, NULL), + VMDIR_SF_INIT(.dwHTTPListenPort, 0), + VMDIR_SF_INIT(.dwHTTPSListenPort, 0), VMDIR_SF_INIT(.dwLdapRecvTimeoutSec, 0), VMDIR_SF_INIT(.dwLdapConnectTimeoutSec, 0), VMDIR_SF_INIT(.mutex, NULL), @@ -93,7 +93,8 @@ VMDIR_GLOBALS gVmdirGlobals = VMDIR_SF_INIT(.dwCopyDbWritesMin, 100), VMDIR_SF_INIT(.dwCopyDbIntervalInSec, 0), VMDIR_SF_INIT(.dwCopyDbBlockWriteInSec, 30), - VMDIR_SF_INIT(.dwLdapWrites, 0) + VMDIR_SF_INIT(.dwLdapWrites, 0), + VMDIR_SF_INIT(.gpVdirSslCtx, NULL) }; VMDIR_KRB_GLOBALS gVmdirKrbGlobals = diff --git a/vmdir/server/vmdir/regconfig.c b/vmdir/server/vmdir/regconfig.c index 84dc785e7..71d300b3f 100644 --- a/vmdir/server/vmdir/regconfig.c +++ b/vmdir/server/vmdir/regconfig.c @@ -169,20 +169,14 @@ VmDirSrvUpdateConfig( VMDIR_REG_KEY_HTTP_LISTEN_PORT, TRUE)) { - dwError = VmDirAllocateStringA( - pEntry->pszValue, - &gVmdirGlobals.pszHTTPListenPort); - BAIL_ON_VMDIR_ERROR(dwError); + gVmdirGlobals.dwHTTPListenPort = pEntry->dwValue; } else if (!VmDirStringCompareA( pEntry->pszName, VMDIR_REG_KEY_HTTPS_LISTEN_PORT, TRUE)) { - dwError = VmDirAllocateStringA( - pEntry->pszValue, - &gVmdirGlobals.pszHTTPSListenPort); - BAIL_ON_VMDIR_ERROR(dwError); + gVmdirGlobals.dwHTTPSListenPort = pEntry->dwValue; } else if (!VmDirStringCompareA( pEntry->pszName, @@ -901,6 +895,93 @@ VmDirGetMaxDbSizeMb( goto cleanup; } +DWORD +VmDirGetMdbWalEnable( + BOOLEAN *pbMdbEnableWal + ) +{ + DWORD keyValue = 1; + DWORD dwError = 0; + PVMDIR_CONFIG_CONNECTION_HANDLE pCfgHandle = NULL; + + if (pbMdbEnableWal==NULL) + { + dwError = VMDIR_ERROR_INVALID_PARAMETER; + BAIL_ON_VMDIR_ERROR(dwError); + } + + *pbMdbEnableWal = FALSE; + + dwError = VmDirRegConfigHandleOpen(&pCfgHandle); + BAIL_ON_VMDIR_ERROR(dwError); + + dwError = VmDirRegConfigGetDword( + pCfgHandle, + VMDIR_CONFIG_PARAMETER_PARAMS_KEY_PATH, + VMDIR_REG_KEY_MDB_ENABLE_WAL, + &keyValue); + BAIL_ON_VMDIR_ERROR(dwError); + + *pbMdbEnableWal = (keyValue!=0); + +cleanup: + if (pCfgHandle) + { + VmDirRegConfigHandleClose(pCfgHandle); + } + return dwError; + +error: + goto cleanup; +} + +DWORD +VmDirGetMdbChkptInterval( + DWORD *pdwMdbChkptInterval + ) +{ + DWORD keyValue = 0; + DWORD dwError = 0; + PVMDIR_CONFIG_CONNECTION_HANDLE pCfgHandle = NULL; + + if (pdwMdbChkptInterval==NULL) + { + dwError = VMDIR_ERROR_INVALID_PARAMETER; + BAIL_ON_VMDIR_ERROR(dwError); + } + + *pdwMdbChkptInterval = VMDIR_REG_KEY_MDB_CHKPT_INTERVAL_DEFAULT; + + dwError = VmDirRegConfigHandleOpen(&pCfgHandle); + BAIL_ON_VMDIR_ERROR(dwError); + + dwError = VmDirRegConfigGetDword( + pCfgHandle, + VMDIR_CONFIG_PARAMETER_PARAMS_KEY_PATH, + VMDIR_REG_KEY_MDB_CHKPT_INTERVAL, + &keyValue); + BAIL_ON_VMDIR_ERROR(dwError); + + if (keyValue < VMDIR_REG_KEY_MDB_CHKPT_INTERVAL_MIN || + keyValue > VMDIR_REG_KEY_MDB_CHKPT_INTERVAL_MAX) + { + dwError = VMDIR_ERROR_INVALID_PARAMETER; + BAIL_ON_VMDIR_ERROR(dwError); + } + + *pdwMdbChkptInterval = keyValue; + +cleanup: + if (pCfgHandle) + { + VmDirRegConfigHandleClose(pCfgHandle); + } + return dwError; + +error: + goto cleanup; +} + DWORD _VmDirDbCpReadRegistry( PDWORD pdwCopyDbWritesMin, diff --git a/vmdir/server/vmdir/rpcserv.c b/vmdir/server/vmdir/rpcserv.c index 7fe705303..f6e9c4fa5 100644 --- a/vmdir/server/vmdir/rpcserv.c +++ b/vmdir/server/vmdir/rpcserv.c @@ -1164,7 +1164,7 @@ _VmDirRemoteDBCopyWhiteList( DWORD dwError = 0; int i = 0; BOOLEAN bAccessAllowed = FALSE; - PSTR pszDBFileNames[] = {"data.mdb", "lock.mdb"}; + PSTR pszDBFileNames[] = {VMDIR_MDB_DATA_FILE_NAME, VMDIR_MDB_LOCK_FILE_NAME, VMDIR_MDB_XLOGS_DIR_NAME}; PSTR pszFullPathName = NULL; #ifdef _WIN32 CHAR pszFilePath[VMDIR_MAX_PATH_LEN] = {0}; @@ -2009,9 +2009,9 @@ void vmdir_dbcp_handle_t_rundown(void *ctx) fclose(pFileHandle); } } - // Clear backend READ-ONLY mode when dbcp connection interrupted. - VmDirSetMdbBackendState(0, &dwXlogNum, &dwDbSizeMb, &dwDbMapSizeMb, tmp_buf, sizeof(tmp_buf)); - VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "vmdir_dbcp_handle_t_rundown: turn off keeping xlog flag on backend, xlognum: %d", dwXlogNum); + // Clear backend READ-ONLY/KeeXlog mode when dbcp connection interrupted. + VmDirSetMdbBackendState(MDB_STATE_CLEAR, &dwXlogNum, &dwDbSizeMb, &dwDbMapSizeMb, tmp_buf, sizeof(tmp_buf)); + VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "vmdir_dbcp_handle_t_rundown: MdbBackendState cleared, xlognum: %d", dwXlogNum); } UINT32 diff --git a/vmdir/server/vmdir/shutdown.c b/vmdir/server/vmdir/shutdown.c index 9f47e023f..1ddf00993 100644 --- a/vmdir/server/vmdir/shutdown.c +++ b/vmdir/server/vmdir/shutdown.c @@ -206,8 +206,6 @@ VmDirCleanupGlobals( // Free vmdir global 'gVmdirGlobals' upon shutdown VMDIR_SAFE_FREE_MEMORY(gVmdirGlobals.pszBDBHome); VMDIR_SAFE_FREE_MEMORY(gVmdirGlobals.pszBootStrapSchemaFile); - VMDIR_SAFE_FREE_MEMORY(gVmdirGlobals.pszHTTPListenPort); - VMDIR_SAFE_FREE_MEMORY(gVmdirGlobals.pszHTTPSListenPort); VMDIR_SAFE_FREE_MUTEX( gVmdirGlobals.replCycleDoneMutex ); VMDIR_SAFE_FREE_MUTEX( gVmdirGlobals.replAgrsMutex ); diff --git a/vmdir/server/vmdir/tenantmgmt.c b/vmdir/server/vmdir/tenantmgmt.c index 91873dd2e..bfb134594 100644 --- a/vmdir/server/vmdir/tenantmgmt.c +++ b/vmdir/server/vmdir/tenantmgmt.c @@ -86,6 +86,12 @@ VmDirSrvCreateTenant( dwError = VmDirDomainNameToDN(pszFQDomainName, &pszDomainDN); BAIL_ON_VMDIR_ERROR(dwError); + // can not create tenant under system domain tree + if (VmDirStringEndsWith(pszDomainDN, gVmdirServerGlobals.systemDomainDN.lberbv_val, FALSE)) + { + BAIL_WITH_VMDIR_ERROR(dwError, VMDIR_ERROR_INSUFFICIENT_ACCESS); + } + dwError = VmDirSchemaCtxAcquire(&pSchemaCtx); BAIL_ON_VMDIR_ERROR(dwError); @@ -113,7 +119,9 @@ VmDirSrvCreateTenant( return dwError; error: - VmDirLog(LDAP_DEBUG_ANY, "VmDirSrvCreateTenantInstance failed. Error(%u)", dwError); + VMDIR_LOG_ERROR(VMDIR_LOG_MASK_ALL, + "%s attempt to create tenant %s failed, error (%u)", + __FUNCTION__, VDIR_SAFE_STRING(pszFQDomainName), dwError); goto cleanup; } @@ -124,8 +132,6 @@ VmDirSrvDeleteTenant( { DWORD dwError = 0; PSTR pszDomainDn = NULL; - PSTR pszParentDn = NULL; - PVDIR_ENTRY pEntry = NULL; VDIR_ENTRY_ARRAY entryArray = {0}; int iIdx = 0; int iCnt = 0; @@ -133,13 +139,17 @@ VmDirSrvDeleteTenant( dwError = VmDirFQDNToDN(pszDomainName, &pszDomainDn); BAIL_ON_VMDIR_ERROR(dwError); - pszParentDn = strchr(pszDomainDn, ','); - if (pszParentDn == NULL) + // can not touch system domain subtree. + if (VmDirStringEndsWith(pszDomainDn, gVmdirServerGlobals.systemDomainDN.lberbv_val, FALSE)) { - BAIL_WITH_VMDIR_ERROR(dwError, VMDIR_ERROR_INVALID_PARAMETER); + BAIL_WITH_VMDIR_ERROR(dwError, VMDIR_ERROR_INSUFFICIENT_ACCESS); } - pszParentDn++; + // ================================================================================ + // Tenant delete IPC makes sure the caller has system domain administrator password + // and has root permission on this VM. + // We need a more controlled way to enforce tenant deletion privilege. + // ================================================================================ dwError = VmDirFilterInternalSearch(pszDomainDn, LDAP_SCOPE_SUBTREE, "objectClass=*", @@ -154,27 +164,15 @@ VmDirSrvDeleteTenant( BAIL_ON_VMDIR_ERROR(dwError); } - // - // Now, try to delete the parent. This can fail if there are other tenants - // still in that root (e.g., we just deleted "pepsi.com" but "coke.com" - // is still around). - // - dwError = VmDirSimpleDNToEntry(pszParentDn, &pEntry); - BAIL_ON_VMDIR_ERROR(dwError); - - dwError = VmDirDeleteEntry(pEntry); - if (dwError == VMDIR_ERROR_NOT_ALLOWED_ON_NONLEAF) - { - dwError = 0; - } - BAIL_ON_VMDIR_ERROR(dwError); - cleanup: VMDIR_SAFE_FREE_STRINGA(pszDomainDn); VmDirFreeEntryArrayContent(&entryArray); - VmDirFreeEntry(pEntry); return dwError; error: + VMDIR_LOG_ERROR(VMDIR_LOG_MASK_ALL, + "%s attempt to delete tenant %s failed, error (%u)", + __FUNCTION__, VDIR_SAFE_STRING(pszDomainName), dwError); + goto cleanup; } diff --git a/vmdir/thirdparty/openldap/libraries/mdb/lmdb.h b/vmdir/thirdparty/openldap/libraries/mdb/lmdb.h index 8a8ebc9a6..565350bac 100644 --- a/vmdir/thirdparty/openldap/libraries/mdb/lmdb.h +++ b/vmdir/thirdparty/openldap/libraries/mdb/lmdb.h @@ -262,6 +262,25 @@ typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); */ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); +/** @brief A callback function invoked before committing a txn + * The transaction will be aborted if this callback return non zero + * Used for Raft implementation to commit a log + */ +typedef int (MDB_raft_prepare_commit_func)(void **raft_commit_ctx); + + +/** @brief A callback function invoked when MDB transaction + * has been succeessfully committed (after obtaining raft consensus). + */ +typedef void (MDB_raft_post_commit_func)(void *raft_commit_ctx); + +/** @brief A callback function invoked when MDB transaction + * fail to flush WAL or write meta page (usually due to disk full/failure) + * The callback should put the server on Raft Follower state so that it will + * not reuse the same logIndex/logTerm for new client request. + */ +typedef void (MDB_raft_commit_fail_func)(void *raft_commit_ctx); + /** @defgroup mdb_env Environment Flags * @{ */ @@ -289,6 +308,8 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel #define MDB_NOMEMINIT 0x1000000 /** keep WAL files after checkpoint -- this version of MDB doesn't support WAL, and the flag is for forward-compatability*/ #define MDB_KEEPXLOGS 0x2000000 + /** Enable WAL (Write Ahead Logging) feature */ +#define MDB_WAL 0x4000000 /** @} */ /** @defgroup mdb_dbi_open Database Flags @@ -412,7 +433,13 @@ typedef enum MDB_cursor_op { #define MDB_BAD_TXN (-30782) /** Too big key/data, key is empty, or wrong DUPFIXED size */ #define MDB_BAD_VALSIZE (-30781) -#define MDB_LAST_ERRCODE MDB_BAD_VALSIZE + /** Corrupted meta page during WAL recover */ +#define MDB_WAL_INVALID_META (-30780) + /** WAL recover failure pages in transaction mismatch */ +#define MDB_WAL_WRONG_TXN_PAGES (-30779) + /** Missing WAL file or invalid WAL file */ +#define MDB_WAL_FILE_ERROR (-30778) +#define MDB_LAST_ERRCODE MDB_WAL_FILE_ERROR /** @} */ /** @brief Statistics for a database in the environment */ @@ -426,6 +453,20 @@ typedef struct MDB_stat { size_t ms_entries; /**< Number of data items */ } MDB_stat; +#define MDB_STATE_OP +/** @brief set, clear or query MDB env state for database file transfer + * MDB_STATE_CLEAR clear MDB_KEEPXLOGS or READONLY state + * MDB_STATE_READONLY - set mdb to READONLY state + * MDB_STATE_KEEPXLOGS - set mdb to keep xlogs state + * MDB_STATE_GETXLOGNUM - query current xlog number + */ +typedef enum MDB_state_op { + MDB_STATE_CLEAR = 0, + MDB_STATE_READONLY, + MDB_STATE_KEEPXLOGS, + MDB_STATE_GETXLOGNUM +} MDB_state_op; + /** @brief Information about the environment */ typedef struct MDB_envinfo { void *me_mapaddr; /**< Address of map, if fixed */ @@ -767,6 +808,13 @@ int mdb_env_set_mapsize(MDB_env *env, size_t size); */ int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); + /** @brief set database checkpoint interval in WAL mode + * + * @param[in] the interval in seconds + * @return A non-zero error value on failure and 0 on success + */ +int mdb_env_set_chkpt_interval(MDB_env *env, int interval); + /** @brief Get the maximum number of threads/reader slots for the environment. * * @param[in] env An environment handle returned by #mdb_env_create() @@ -1119,6 +1167,11 @@ int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); */ int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); + /** @brief Set commit hook func for Raft + * + */ +void mdb_set_raft_prepare_commit_func(MDB_env *env, MDB_raft_prepare_commit_func *raft_prepare_commit_func); + /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. * * See #mdb_set_relfunc and #MDB_rel_func for more details. @@ -1133,6 +1186,11 @@ int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); *
  • EINVAL - an invalid parameter was specified. * */ + + /** @brief callback for raft post commit - set raft volatle state with logIndex argument when commit succeeded + */ +void mdb_set_raft_post_commit_func(MDB_env *env, MDB_raft_post_commit_func *raft_post_commit_func); + int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); /** @brief Get items from a database. @@ -1460,25 +1518,21 @@ int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); */ int mdb_reader_check(MDB_env *env, int *dead); - /** @brief set or clear database file transfer state for remote file copy. - * @param[in] env - environment handle returned by #mdb_env_create() - * @param[in] 1 - set mdb to READONLY state - * 2 - set mdb to keep xlogs state (used for hot database file copy only) - * 0 - clear MDB_KEEPXLOGS flag or mdb READONLY state - * 3 - don't change mdb state, only return database sizes, path, etc. - * @param[out] The starting transaction log number if dwFileTransferState is 2 and mdb support WAL. - * If dwFileTransferState is 2 but mdb doesn't support WAL, then mdb would - * be put at READONLY mode, and pdwLogNum set to 0. - * @param[out] assigned size of the partner's backend database file in MB. - * @param[out] the map size of the partner's backend database file in MB. - * @param[out] the path of the database home (where data.mdb and lock.mdb located) - * @param[in] the memory size of db_path. + /** @brief set, clear or query MDB state for database file cold or hop copy. + * @param[in] env - environment handle returned by #mdb_env_create() + * @param[in] op - MDB_state_op + * @param[out] last_xlog_num - set to the current WAL log numbern if MDB supports WAL + * otherwise set to 0. + * @param[out] dbSizeMb - allocated database size in MB (round to the next MB). + * @param[out] dbMapSizeMb - the database map size in MB. + * @param[out] db_path - the path of the database home where data.mdb and lock.mdb resides + * @param[in] db_path_size - the memory size of db_path. * @return 0 success - * 1 failed - invalid environment handle - * 2 failed - invalid state - * 3 failed - db_path buffer too small + * EINVAL invalid environment handle, input parameter or state + * EOVERFLOW db_path buffer too small */ -int mdb_env_set_state(MDB_env *env, int fileTransferState, unsigned long *last_xlog_num, unsigned long *dbSizeMb, unsigned long *dbMapSizeMb, char *db_path, int db_path_size); +int mdb_env_set_state(MDB_env *env, MDB_state_op op, unsigned long *last_xlog_num, unsigned long *dbSizeMb, + unsigned long *dbMapSizeMb, char *db_path, int db_path_size); unsigned long long mdb_env_get_lasttid(MDB_env *env); diff --git a/vmdir/thirdparty/openldap/libraries/mdb/mdb.c b/vmdir/thirdparty/openldap/libraries/mdb/mdb.c index fd0d46b5d..eb60f9a70 100644 --- a/vmdir/thirdparty/openldap/libraries/mdb/mdb.c +++ b/vmdir/thirdparty/openldap/libraries/mdb/mdb.c @@ -77,6 +77,7 @@ #include #ifndef _WIN32 #include +#include #endif #if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) @@ -316,6 +317,10 @@ mdb_sem_wait(sem_t *sem) #ifndef MDB_DSYNC # define MDB_DSYNC O_DSYNC #endif +/** + * initial and incremental database size in Bytes - 256MB + */ +#define DB_SIZE_INC (1LL << 28) #endif /** Function for flushing the data of a file. Define this to fsync @@ -890,6 +895,14 @@ typedef struct MDB_meta { #define mm_flags mm_dbs[0].md_flags pgno_t mm_last_pg; /**< last used page in file */ txnid_t mm_txnid; /**< txnid that committed this page */ + /** number of pages of this transaction not including the meta page used for WAL auditing */ + uint32_t mm_txn_pages; + /** The last xlog num being used. + * If the server was shutdown gracefully and all xlog files were purged, + * then this number is taken in database file's meta data, otherwise + * the meta data is rollforwarded by txn log files. */ + uint32_t mm_xlog_num; + uint32_t mm_xlog_num_pre_chkpt; } MDB_meta; /** Buffer for a stack-allocated meta page. @@ -1058,6 +1071,34 @@ typedef struct MDB_pgstate { txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ } MDB_pgstate; + /** State of write ahead logging **/ + /** maximum number of pages before trying to use the next WAL file */ +#define MAX_WAL_PGS 32768 + /** initial WAL buffer pages */ +#define WAL_INIT_PGS (MAX_WAL_PGS >> 3) +#define XLOG_MIN_NUM 10000001 +#define XLOG_MAX_NUM 99999999 + +/* purge upto current xlog_num file less the margin */ +#define XLOG_PURGE_SAFE_MARGIN 5 + +/* the default value of check point interval */ +#define CHKPT_INTERVAL_DEFAULT 30 + +typedef struct MDB_walstate { + pthread_t chkpt_thread; /* check point thread id */ + unsigned long xlog_num; /* current WAL file number that derives the file name */ + unsigned long xlog_num_pre_chkpt; /* WAL file number right before chkpt completed */ + unsigned long xlog_purged; /* last WAL file number being purged */ + HANDLE xlog_fd; /* current WAL file fd */ + uint32_t xlog_pages; /* number of pages written to current WAL file */ + uint32_t chkpt_interval; /* the interval in seconds of doing checkpoint on database */ + pthread_cond_t chkpt_waitcond; /* for waking up check point thread */ + pthread_mutex_t chkpt_waitmutex;/* Mutex for check point thread */ + unsigned long txn_pages; /* number of pages to write to wal for current transaction */ + int chkpt_thread_active; /* set to 1 when chkpt_thread started */ +} MDB_walstate; + /** The database environment. */ struct MDB_env { HANDLE me_fd; /**< The main data file */ @@ -1086,7 +1127,7 @@ struct MDB_env { void *me_pbuf; /**< scratch area for DUPSORT put() */ MDB_txn *me_txn; /**< current write transaction */ size_t me_mapsize; /**< size of the data memory map */ -// off_t me_size; /**< current file size */ + off_t me_size; /**< current file size */ pgno_t me_maxpg; /**< me_mapsize / me_psize */ MDB_dbx *me_dbxs; /**< array of static DB info */ uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ @@ -1118,6 +1159,10 @@ struct MDB_env { #endif void *me_userctx; /**< User-settable context */ MDB_assert_func *me_assert_func; /**< Callback for assertion failures */ + MDB_raft_prepare_commit_func *me_raft_prepare_commit_func; /** Commit hook function used for Raft committing a log **/ + MDB_raft_post_commit_func *me_raft_post_commit_func; /** callback that sets raft state for the logIndex **/ + MDB_raft_commit_fail_func *me_raft_commit_fail_func; /** callback that sets raft state if fail to write WAL or meta page **/ + MDB_walstate me_walstate; /** WAL (write ahead logging) state **/ }; /** Nested transaction */ @@ -1193,7 +1238,15 @@ static void mdb_xcursor_init0(MDB_cursor *mc); static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); static int mdb_drop0(MDB_cursor *mc, int subs); -static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); +static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); +static int mdb_wal_init(MDB_env *env); +static int mdb_rollxlogs(MDB_env *env, int purge); +static int mdb_rollforward_file(MDB_env *env, char * xlog_file); +static void * mdb_chkpt_main(void *param_ptr); +#ifndef _WIN32 +static int wal_sync_meta(MDB_env *env, txnid_t tid); +static int write_wal_pages(MDB_env *env, const struct iovec *iov, int n); +#endif /** @cond */ static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; @@ -1221,7 +1274,7 @@ static char *const mdb_errstr[] = { "MDB_NOTFOUND: No matching key/data pair found", "MDB_PAGE_NOTFOUND: Requested page not found", "MDB_CORRUPTED: Located page was wrong type", - "MDB_PANIC: Update of meta page failed", + "MDB_PANIC: Update of meta page or WAL file failed", "MDB_VERSION_MISMATCH: Database environment version mismatch", "MDB_INVALID: File is not an MDB file", "MDB_MAP_FULL: Environment mapsize limit reached", @@ -1236,6 +1289,9 @@ static char *const mdb_errstr[] = { "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", "MDB_BAD_TXN: Transaction cannot recover - it must be aborted", "MDB_BAD_VALSIZE: Too big key/data, key is empty, or wrong DUPFIXED size", + "MDB_WAL_INVALID_META: WAL recover failure - invalid meta page or missing WAL file", + "MDB_WAL_WRONG_TXN_PAGES: WAL recover failure - pages in transaction mismatch", + "MDB_WAL_FILE_ERROR: Missing or bad WAL file" }; char * @@ -2864,8 +2920,13 @@ mdb_page_flush(MDB_txn *txn, int keep) /* Write up to MDB_COMMIT_PAGES dirty pages at a time. */ if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) { if (n) { - /* Write previous page(s) */ + /* write pages to WAL file */ + rc = write_wal_pages(env, iov, n); + if (rc) + return rc; #ifdef MDB_USE_PWRITEV + + /* Write previous page(s) */ wres = pwritev(env->me_fd, iov, n, wpos); #else if (n == 1) { @@ -2929,6 +2990,7 @@ mdb_txn_commit(MDB_txn *txn) int rc; unsigned int i; MDB_env *env; + void *raft_commit_ctx = NULL; if (txn == NULL || txn->mt_env == NULL) return EINVAL; @@ -3107,17 +3169,33 @@ mdb_txn_commit(MDB_txn *txn) #if (MDB_DEBUG) > 2 mdb_audit(txn); #endif - - if ((rc = mdb_page_flush(txn, 0)) || - (rc = mdb_env_sync(env, 0)) || - (rc = mdb_env_write_meta(txn))) - goto fail; + if ((rc = mdb_page_flush(txn, 0))) + { + goto fail; + } + + if (!(env->me_flags & MDB_WAL)){ + if((rc = mdb_env_sync(env, 0))) + goto fail; + } + + if ((env->me_raft_prepare_commit_func && + (rc = env->me_raft_prepare_commit_func(&raft_commit_ctx))) || + (rc = mdb_env_write_meta(txn))) + { + goto fail; + } done: env->me_pglast = 0; env->me_txn = NULL; mdb_dbis_update(txn, 1); + if (env->me_raft_post_commit_func) + { + env->me_raft_post_commit_func(raft_commit_ctx); + } + if (env->me_txns) UNLOCK_MUTEX_W(env); free(txn); @@ -3125,6 +3203,10 @@ mdb_txn_commit(MDB_txn *txn) return MDB_SUCCESS; fail: + if (env->me_raft_commit_fail_func) + { + env->me_raft_commit_fail_func(raft_commit_ctx); + } mdb_txn_abort(txn); return rc; } @@ -3231,6 +3313,8 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta) meta->mm_flags |= MDB_INTEGERKEY; meta->mm_dbs[0].md_root = P_INVALID; meta->mm_dbs[1].md_root = P_INVALID; + meta->mm_xlog_num = XLOG_MIN_NUM - 1; + meta->mm_xlog_num_pre_chkpt = XLOG_MIN_NUM; p = calloc(2, psize); p->mp_pgno = 0; @@ -3266,6 +3350,11 @@ mdb_env_write_meta(MDB_txn *txn) int rc, len, toggle; char *ptr; HANDLE mfd; + MDB_metabuf mbuf = {0}; + MDB_page *dp; + MDB_page *np = NULL; + int nw = 0; + #ifdef _WIN32 OVERLAPPED ov; #else @@ -3278,6 +3367,11 @@ mdb_env_write_meta(MDB_txn *txn) env = txn->mt_env; mp = env->me_metas[toggle]; + dp = (MDB_page *)env->me_map; + if (toggle) + dp = (MDB_page *)(env->me_map + env->me_psize); + memcpy(&mbuf.mb_page, (char *)dp, PAGEHDRSZ); + if (env->me_flags & MDB_WRITEMAP) { /* Persist any increases of mapsize config */ if (env->me_mapsize > mp->mm_mapsize) @@ -3319,10 +3413,48 @@ mdb_env_write_meta(MDB_txn *txn) len = sizeof(MDB_meta) - off; ptr += off; - meta.mm_dbs[0] = txn->mt_dbs[0]; - meta.mm_dbs[1] = txn->mt_dbs[1]; - meta.mm_last_pg = txn->mt_next_pgno - 1; - meta.mm_txnid = txn->mt_txnid; + mbuf.mb_metabuf.mm_meta.mm_magic = mp->mm_magic; + mbuf.mb_metabuf.mm_meta.mm_version = mp->mm_version; + mbuf.mb_metabuf.mm_meta.mm_address = mp->mm_address; + mbuf.mb_metabuf.mm_meta.mm_dbs[0] = meta.mm_dbs[0] = txn->mt_dbs[0]; + mbuf.mb_metabuf.mm_meta.mm_dbs[1] = meta.mm_dbs[1] = txn->mt_dbs[1]; + mbuf.mb_metabuf.mm_meta.mm_last_pg = meta.mm_last_pg = txn->mt_next_pgno - 1; + mbuf.mb_metabuf.mm_meta.mm_txnid = meta.mm_txnid = txn->mt_txnid; + mbuf.mb_metabuf.mm_meta.mm_txn_pages = meta.mm_txn_pages = env->me_walstate.txn_pages; + mbuf.mb_metabuf.mm_meta.mm_xlog_num = meta.mm_xlog_num = env->me_walstate.xlog_num; + mbuf.mb_metabuf.mm_meta.mm_xlog_num_pre_chkpt = meta.mm_xlog_num_pre_chkpt = env->me_walstate.xlog_num_pre_chkpt; + env->me_walstate.txn_pages = 0; + +#ifndef _WIN32 + if (env->me_flags & MDB_WAL) + { + np = mdb_page_malloc(txn, 1); + if (np == NULL) + { + rc = ENOMEM; + goto fail; + } + + memcpy(np, (char *)&mbuf, sizeof(MDB_metabuf)); + nw = write(env->me_walstate.xlog_fd, np, env->me_psize); + mdb_page_free(env, np); + + if (nw != env->me_psize) + { + if (nw < 0) + rc = ErrCode(); + else + rc = ENOMEM; + goto fail; + } + env->me_walstate.xlog_pages++; + + if (wal_sync_meta(env, txn->mt_txnid) != 0) + { + goto fail; + } + } +#endif if (toggle) off += env->me_psize; @@ -3339,7 +3471,7 @@ mdb_env_write_meta(MDB_txn *txn) rc = -1; } #else - rc = pwrite(mfd, ptr, len, off); + rc = pwrite(env->me_fd, ptr, len, off); #endif if (rc != len) { rc = rc < 0 ? ErrCode() : EIO; @@ -3536,6 +3668,16 @@ mdb_env_set_maxreaders(MDB_env *env, unsigned int readers) return MDB_SUCCESS; } +int +mdb_env_set_chkpt_interval(MDB_env *env, int interval) +{ + if (env == NULL) + return EINVAL; + + env->me_walstate.chkpt_interval = interval; + return MDB_SUCCESS; +} + int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) { @@ -4110,9 +4252,9 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) * at runtime. Changing other flags requires closing the * environment and re-opening it with the new flags. */ -#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) +#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT|MDB_KEEPXLOGS) #define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP| \ - MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) + MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_WAL) #if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS) # error "Persistent DB flags & env flags overlap, but both go in mm_flags" @@ -4124,9 +4266,14 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode int oflags, rc, len, excl = -1; char *lpath, *dpath; - if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) + if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS)) || + ((flags & MDB_WAL) && (flags & MDB_WRITEMAP))) return EINVAL; - +#ifdef _WIN32 + //WAL feature is currently not supported on Windows + if (flags & MDB_WAL) + return EINVAL; +#endif len = (int) strlen(path); if (flags & MDB_NOSUBDIR) { rc = len + sizeof(LOCKSUFF) + len + 1; @@ -4236,11 +4383,25 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode (env->me_pbuf = calloc(1, env->me_psize)))) rc = ENOMEM; } + if (env->me_flags & MDB_WAL) + { + if ((rc=mdb_rollxlogs(env, 0)) != MDB_SUCCESS) + { + goto done; + } + + if ((rc=mdb_wal_init(env) != MDB_SUCCESS)) + { + goto leave; + } + } leave: if (rc) { mdb_env_close0(env, excl); } + +done: free(lpath); return rc; } @@ -4254,10 +4415,41 @@ mdb_env_close0(MDB_env *env, int excl) if (!(env->me_flags & MDB_ENV_ACTIVE)) return; + if (env->me_flags & MDB_WAL) + { + env->me_flags &= ~MDB_ENV_ACTIVE; + pthread_cond_signal(&env->me_walstate.chkpt_waitcond); + if (env->me_walstate.chkpt_thread_active) + pthread_join(env->me_walstate.chkpt_thread, NULL); + + if (env->me_walstate.xlog_fd != INVALID_HANDLE_VALUE) + { + close(env->me_walstate.xlog_fd); + env->me_walstate.xlog_fd = INVALID_HANDLE_VALUE; + } + + if (!(env->me_flags & MDB_FATAL_ERROR) && + mdb_env_sync(env, 1) == 0 && + !(env->me_flags & MDB_KEEPXLOGS)) + { + /* Don't purge WAL files and sync database if a fatal error condition exists, + * and go through WAL recovery procedure when server restarts. + * Don't purge WAL files if sync database failed, and go through WAL + * recovery procedure when server restarts. + */ + mdb_rollxlogs(env, 1); + } + } + /* Doing this here since me_dbxs may not exist during mdb_env_close */ for (i = env->me_maxdbs; --i > MAIN_DBI; ) free(env->me_dbxs[i].md_name.mv_data); + if (env->me_flags & MDB_WAL) + { + pthread_mutex_destroy(&env->me_walstate.chkpt_waitmutex); + pthread_cond_destroy(&env->me_walstate.chkpt_waitcond); + } free(env->me_pbuf); free(env->me_dbflags); free(env->me_dbxs); @@ -8544,50 +8736,709 @@ int mdb_reader_check(MDB_env *env, int *dead) return MDB_SUCCESS; } -/** - * lmdb.h mdb_env_set_state for parameters +/* + * Rollfoward pages assocated with a single transaction + * with pages started from "start" to "end" in that order, + * with memory pointers stored in "xlog_pgs". + * Page pointed by "end" is always a meta page + * Each transaction is limitted to (2^17 - 1) pages + * of data (e.g. 512MB when page size is 4Kb) + * The limitation can be increased by extending + * the compile time MDB_IDL_UM_SIZE value + */ +static +int commit_xlog_txn(MDB_env *env, MDB_ID2L xlog_pgs, int start, int end) +{ + MDB_page *p; + MDB_metabuf *mbufp; + MDB_meta *m; + int i, j; + + mbufp = (MDB_metabuf *)xlog_pgs[end].mptr; + m = &mbufp->mb_metabuf.mm_meta; + if (m->mm_magic != MDB_MAGIC) + return MDB_WAL_INVALID_META; + if ((end - start) != m->mm_txn_pages) + return MDB_WAL_WRONG_TXN_PAGES; + + i = start; + while(i <= end) + { + p = xlog_pgs[i].mptr; + if (IS_OVERFLOW(p)) + { + for (j=0; j < (int)p->mp_pages; j++) { + unsigned long d_offset = (unsigned long)(((p->mp_pgno +j ) * env->me_psize)); + char *s_pos = xlog_pgs[i+j].mptr; + pwrite(env->me_fd, s_pos, env->me_psize, d_offset); + } + mdb_eassert(env, p->mp_pages > 0); + i += p->mp_pages; + } else if (F_ISSET(p->mp_flags, P_META)) + { + mbufp = (MDB_metabuf *)p; + m = &mbufp->mb_metabuf.mm_meta; + pwrite(env->me_mfd, p, env->me_psize, (p->mp_pgno * env->me_psize)); + env->me_txns->mti_txnid = m->mm_txnid; + i++; + } else + { + pwrite(env->me_fd, p, env->me_psize, (p->mp_pgno * env->me_psize)); + i++; + } + } + return 0; +} + +#ifdef _WIN32 +#define UNLINK_FILE(s) _unlink(s) +#else +#define UNLINK_FILE(s) unlink(s) +#endif + +/* + * Rollfoward a single transaction log files, xlog_file + * Each xlog_file may contain pages cover multiple tranactions, + * but no transaction is allowed to across more than one xlog file. + */ +static +int mdb_rollforward_file(MDB_env *env, char * xlog_file) +{ + HANDLE fd = INVALID_HANDLE_VALUE; + int rc = 0, nr, i, j, cnt; + char *p = NULL; + MDB_page *mp; + MDB_ID2L xlog_pgs = NULL; + MDB_ID2 mid; +#ifdef _WIN32 + DWORD len; +#endif + xlog_pgs = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2)); + if (xlog_pgs == NULL) + { + rc = ENOMEM; + goto done; + } +#ifdef _WIN32 + fd = CreateFile(xlog_file, GENERIC_READ, FILE_SHARE_READ, + NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); +#else + fd = open(xlog_file, O_RDONLY); +#endif + if (fd == INVALID_HANDLE_VALUE ) + { + rc = ErrCode(); + goto cleanup; + } + p = aligned_alloc(env->me_psize, env->me_psize); + if (p == NULL) + { + rc = ENOMEM; + goto cleanup; + } + while(1) + { +#ifdef _WIN32 + nr = ReadFile(fd, p, env->me_psize, &len, NULL) ? (int)len : -1; +#else + nr = read(fd, p, env->me_psize); +#endif + if (nr != (int)env->me_psize) + break; + mp = (MDB_page *)p; + mid.mid = mp->mp_pgno; + mid.mptr = mp; + rc = mdb_mid2l_append(xlog_pgs, &mid); + if (rc) + { + rc = ENOMEM; + goto cleanup; + } + p = aligned_alloc(env->me_psize, env->me_psize); + if (p == NULL) + { + rc = ENOMEM; + goto cleanup; + } + } + + if (nr == 0) + { + i = 1; + j = 1; + cnt = (int)xlog_pgs[0].mid; + while (i <= cnt) + { + mp = xlog_pgs[i].mptr; + if (IS_OVERFLOW(mp)){ + //printf("read %lu overflow pages on pgno %lu\n", mp->mp_pages, mp->mp_pgno); + i += mp->mp_pages; + } + else if (F_ISSET(mp->mp_flags, P_META)) + { + //printf("commiting file %s from %d to %d (%d pgs)\n", xlog_file, j, i, i-j+1); + rc = commit_xlog_txn(env, xlog_pgs, j, i); + if (rc) + goto cleanup; + i++; + j = i; + } else + i++; + } + goto cleanup; + } else if (nr < 0) + { + rc = ErrCode(); + goto cleanup; + } else { + rc = MDB_WAL_WRONG_TXN_PAGES; + goto cleanup; + } +cleanup: + if (p) + free(p); + i = 0; + cnt = (int) xlog_pgs[0].mid; + while (++i <= cnt) { + mp = xlog_pgs[i].mptr; + free(mp); + } + free(xlog_pgs); + if (fd >= 0) + close(fd); +done: + if (rc == 0 ) + { + if((rc=mdb_env_sync(env, 1)) != 0) + { + return rc; + } + if (!(env->me_flags & MDB_KEEPXLOGS)) + { + UNLINK_FILE(xlog_file); + } + } + return rc; +} + +/* Rollforward WAL files to the memory associated + * with the memory mapped database file if purge is 0, + * othewise purge all WAL files. Rollfoward is triggered + * if there is any WAL files in the destinated directory. + * Usually all WAL files were purged during graceful shutdown + * except when env is opended with MDB_KEEPXLOGS flag + * + * This function also discovers missing xlog files, + * and skip xlog files that are older than database file. + */ +static +int mdb_rollxlogs(MDB_env *env, int purge) +{ + char xlog_file[256]; + char xlog_file_dir[256]; + char xlog_bad_file[256]; + unsigned long xlog_num = 0, i; + int rc = 0; + unsigned long mm_xlog_num_pre_chkpt = env->me_metas[mdb_env_pick_meta(env)]->mm_xlog_num_pre_chkpt; + +#ifdef _WIN32 + WIN32_FIND_DATA ffd; + HANDLE d = INVALID_HANDLE_VALUE; +#else + DIR *d = NULL; + struct dirent *dir = NULL; +#endif + + MDB_IDL xlog_ids = mdb_midl_alloc(MDB_IDL_UM_MAX); + if(!xlog_ids) + { + rc = ENOMEM; + goto done; + } + +#ifdef _WIN32 + sprintf(xlog_file_dir, "%s\\xlogs\\1*", env->me_path); + d = FindFirstFile(xlog_file_dir, &ffd); + if (d == INVALID_HANDLE_VALUE) + { + rc = ErrCode(); + if (rc == ERROR_FILE_NOT_FOUND || rc == ERROR_PATH_NOT_FOUND) + rc = 0; + else + rc = ENOMEM; + goto done; + } + if (!(ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + { + xlog_num=atol(ffd.cFileName); + if(xlog_num>=XLOG_MIN_NUM && xlog_num <= XLOG_MAX_NUM) + { + mdb_midl_xappend(xlog_ids, xlog_num); + } + } + while (1) + { + rc = FindNextFile(d, &ffd); + if (rc == 0) + { + rc=ErrCode(); + if (rc == ERROR_NO_MORE_FILES) + { + rc = 0; + break; + } + else + { + rc = ENOMEM; + goto done; + } + } else if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + continue; + else + xlog_num=atol(ffd.cFileName); +#else + sprintf(xlog_file_dir, "%s/xlogs", env->me_path); + d = opendir(xlog_file_dir); + if (d == NULL ) + { + if(errno == ENOENT ) + rc = 0; + else + rc = ENOMEM; + goto done; + } + + while ((dir = readdir(d)) != NULL) + { + xlog_num=atol(dir->d_name); +#endif + if(xlog_num>=XLOG_MIN_NUM && xlog_num <= XLOG_MAX_NUM) + { + if (xlog_ids[0] > (MDB_IDL_UM_MAX - 2)) + { + rc = ENOMEM; + goto done; + } + mdb_midl_xappend(xlog_ids, xlog_num); + } + } + mdb_midl_sort(xlog_ids); + i = (unsigned long)xlog_ids[0]; + if (!purge && i > 0) + { + if ((xlog_ids[1] - xlog_ids[i] + 1) != i || //logs are not continous numbers (missing files) + xlog_ids[1] < mm_xlog_num_pre_chkpt) //the last log is older than database + { + //Missing one or more WAL files + rc = MDB_WAL_FILE_ERROR; + mdb_eassert(env, rc == 0); + goto done; + } + DPRINTF(("MDB recover is needed; roll forward %ld transaction log files...", i)); + } + + for (; i; i--) + { +#ifdef _WIN32 + sprintf(xlog_file, "%s\\xlogs\\%08lu", env->me_path, xlog_ids[i]); +#else + sprintf(xlog_file, "%s/%08lu", xlog_file_dir, xlog_ids[i]); +#endif + if (purge) + { + rc = UNLINK_FILE(xlog_file); + //If unlink failed, the next startup may roll forward it, compromising integrity + mdb_eassert(env, rc == 0); + continue; + } + + if ((xlog_ids[i] + 1)< mm_xlog_num_pre_chkpt) + { + //Database is newer than the xlog file. + DPRINTF(("skip rollfoward xlog_file %s\n", xlog_file)); + if (!(env->me_flags & MDB_KEEPXLOGS)) + UNLINK_FILE(xlog_file); + continue; + } + + DPRINTF(("rollfoward xlog_file %s\n", xlog_file)); + rc = mdb_rollforward_file(env, xlog_file); + if (rc) + { +#ifndef _WIN32 + sprintf(xlog_bad_file, "%s/garbled-%08lu", xlog_file_dir, xlog_ids[i]); + rename(xlog_file, xlog_bad_file); +#endif + if (i==1) + { + //This failure is most likely recoverable, and the garbled file was due + // to machine powered off before fdatasync completed on the WAL file, + // leaving the incompleted (the last) transaction in the WAL file. + rc = 0; + continue; + } + else + //This is usually not recoverable, namely a WAL is inconsistent even + //though it has completed fdatasync, and proceeded to the next log file. + mdb_eassert(env, rc == 0); + goto done; + } + } + +done: +#ifdef _WIN32 + if (d != INVALID_HANDLE_VALUE) + FindClose(d); +#else + if (d) + closedir(d); +#endif + mdb_midl_free(xlog_ids); + return rc; +} + +/* Initialize WAL state structure. + * Called once when the environment is opened + */ +static +int mdb_wal_init(MDB_env *env) +{ +#ifdef _WIN32 + DWORD dwAttrib; +#endif + char xlog_dir[256]; + int rc = 0; + + env->me_walstate.xlog_fd = INVALID_HANDLE_VALUE; + env->me_walstate.xlog_num = env->me_metas[mdb_env_pick_meta(env)]->mm_xlog_num + 1; + if (env->me_walstate.xlog_num < XLOG_MIN_NUM) + //This may occur once when switching from no-wal data.mdb to wal data.mdb + env->me_walstate.xlog_num = XLOG_MIN_NUM; + env->me_walstate.xlog_purged = XLOG_MIN_NUM - 1; + env->me_walstate.xlog_pages = 0; + env->me_walstate.txn_pages = 0; + if (env->me_walstate.chkpt_interval == 0) + { + //If mdb_env_set_chkpt_interval is not called, set to default value. + env->me_walstate.chkpt_interval = CHKPT_INTERVAL_DEFAULT; + } + if ((rc=pthread_mutex_init(&env->me_walstate.chkpt_waitmutex, NULL)) || + (rc=pthread_cond_init(&env->me_walstate.chkpt_waitcond, NULL))) + return rc; + +#ifdef _WIN32 + sprintf(xlog_dir, "%s\\xlogs", env->me_path); + dwAttrib = GetFileAttributes(xlog_dir); + if(dwAttrib == INVALID_FILE_ATTRIBUTES && CreateDirectory(xlog_dir, NULL)==0) +#else + sprintf(xlog_dir, "%s/xlogs", env->me_path); + rc = access(xlog_dir, R_OK|W_OK|X_OK); + if(rc != 0 && mkdir(xlog_dir, 0700)!=0) +#endif + return ENOMEM; + + if ((rc=pthread_create(&env->me_walstate.chkpt_thread, NULL, mdb_chkpt_main, (void *)env))) + { + env->me_walstate.chkpt_thread_active = 0; + return rc; + } + env->me_walstate.chkpt_thread_active = 1; + return MDB_SUCCESS; +} + +#ifndef _WIN32 +/* The all pages associated with the transaction + * into the WAL file. The last pages should always + * the meta page. Try to use O_DIRECT if the the OS/FS + * support it, otherwise issue an fdatasync to the + * WAL file. Close the WAL if its size exceeded the defined + * limit */ +static +int wal_sync_meta(MDB_env *env, txnid_t tid) +{ + char xlog_file[256]; + int rc = 0; + int i = 0; + HANDLE fd; + + for(i=0; i<3; i++) + { + rc = fdatasync(env->me_walstate.xlog_fd); + if (rc == 0) + break; + } + + if (rc) + { + goto done; + } + + if (env->me_walstate.xlog_pages >= MAX_WAL_PGS) + { + mdb_eassert(env, env->me_walstate.xlog_fd != INVALID_HANDLE_VALUE); + close(env->me_walstate.xlog_fd); + env->me_walstate.xlog_fd = INVALID_HANDLE_VALUE; + env->me_walstate.xlog_num++; + env->me_walstate.xlog_pages = 0; + env->me_walstate.txn_pages = 0; + sprintf(xlog_file, "%s/xlogs/%08lu", env->me_path, env->me_walstate.xlog_num); + fd = open(xlog_file, O_CREAT|O_WRONLY, S_IRUSR|S_IWUSR); + if (fd == INVALID_HANDLE_VALUE) + { + rc = ENOMEM; + goto done; + } + env->me_walstate.xlog_fd = fd; + } + +done: + return rc; +} +#endif + +/* The checkpoint thread main function + * performs msync in a fixed interval, and + * then purge the transaction log files that are + * no longer needed for database recover. + * If MDB_KEEPXLOGS is set, no purge + * will be done, and the transaction log + * files, plus an earlier database file backup, + * can be used to restore the database to + * an earlier state (between when the database + * file was backup and the time the last log file + * was created). + */ +static +void *mdb_chkpt_main(void *param_ptr) +{ + long i, rc=0; + char xlog_file_to_purge[256]; + long xlog_num_pre_chkpt; + time_t now; + struct timespec ts; + int fatal_error = 0; + + MDB_env *env = (MDB_env *)param_ptr; + + while (env->me_flags & MDB_ENV_ACTIVE) + { + LOCK_MUTEX_W(env); + if (env->me_flags & MDB_FATAL_ERROR) + fatal_error = 1; //This can occur if disk is full when writing WAL file + else + { + xlog_num_pre_chkpt = env->me_walstate.xlog_num; + if (xlog_num_pre_chkpt < XLOG_MIN_NUM) + //This can occur once when switching from no-wal mdb to wal mdb + xlog_num_pre_chkpt = XLOG_MIN_NUM; + + DPRINTF(("mdb_chkpt_main calls mdb_env_sync ...")); + for(i=0; i<3; i++) + { + rc = mdb_env_sync(env, 1); + if (rc == 0) + break; + } + if (rc) + { + env->me_flags |= MDB_FATAL_ERROR; + fatal_error = 1; + } else + { + env->me_walstate.xlog_num_pre_chkpt = xlog_num_pre_chkpt; + } + } + UNLOCK_MUTEX_W(env); + + if (!fatal_error && !(MDB_KEEPXLOGS & env->me_flags)) + { + /* Any closed xlog files before chkpt can be safely purged + * - keep a few more files as a safe margin + */ + for (i=env->me_walstate.xlog_purged + 1; i<(xlog_num_pre_chkpt - XLOG_PURGE_SAFE_MARGIN); i++) + { + sprintf(xlog_file_to_purge, "%s/xlogs/%08lu", env->me_path, i); + if (UNLINK_FILE(xlog_file_to_purge) == 0) + env->me_walstate.xlog_purged = i; + } + } + + if ((env->me_flags & MDB_ENV_ACTIVE) == 0) + return NULL; //env has been shutdown + + now = time(NULL); + ts.tv_sec = now + env->me_walstate.chkpt_interval; + ts.tv_nsec = 0; + pthread_mutex_lock(&env->me_walstate.chkpt_waitmutex); + pthread_cond_timedwait(&env->me_walstate.chkpt_waitcond, + &env->me_walstate.chkpt_waitmutex, &ts); + pthread_mutex_unlock(&env->me_walstate.chkpt_waitmutex); + } + DPRINTF(("mdb_chkpt_main exits.")); + return NULL; +} + +/** @brief set, clear or query MDB state for database file cold or hot copy. + * Refer its description in lmdb.h for parameters. */ int -mdb_env_set_state(MDB_env *env, int fileTransferState, unsigned long *last_xlog_num, unsigned long *dbSizeMb, unsigned long *dbMapSizeMb, char *db_path, int db_path_size) +mdb_env_set_state(MDB_env *env, MDB_state_op op, unsigned long *last_xlog_num, unsigned long *dbSizeMb, + unsigned long *dbMapSizeMb, char *db_path, int db_path_size) { MDB_envinfo env_stats = {0}; int ret = 0; if (env == NULL) - return 1; // invalid parameter + return EINVAL; - *last_xlog_num = 0; //Indicating that the backend doesn't support write-ahead-logging (WAL). - //Will try to put the backend onto read-only state. + *last_xlog_num = 0; LOCK_MUTEX_W(env); - if (fileTransferState != 3) + if (((env->me_flags & MDB_RDONLY) && op == MDB_STATE_READONLY) || + //Already in read-only state while trying to set to read-only + ((env->me_flags & MDB_RDONLY) && op == MDB_STATE_KEEPXLOGS) || + //Already in read-only state while trying to set to keep XLOGS + (!(env->me_flags & MDB_RDONLY) && !(env->me_flags & MDB_KEEPXLOGS) && op == MDB_STATE_CLEAR) + // Not in read-only state while trying to clear read-only or keep XLOGS state + ) + ret = EINVAL; + else if ( op == MDB_STATE_READONLY) + env->me_flags |= MDB_RDONLY; + else if (op == MDB_STATE_KEEPXLOGS) { - if (((env->me_flags & MDB_RDONLY) && fileTransferState == 1) || //Already in read-only state while trying to set to read-only - ((env->me_flags & MDB_RDONLY) && fileTransferState == 2) || //Already in read-only state while trying to set to keep XLOGS - (!(env->me_flags & MDB_RDONLY) && fileTransferState == 0)) // Not in read-only state while trying to clear read-only or keep XLOGS state - ret = 2; - else if ( fileTransferState == 1 || fileTransferState == 2 ) + if (env->me_flags & MDB_WAL) + { + *last_xlog_num = env->me_walstate.xlog_num; + env->me_flags |= MDB_KEEPXLOGS; + } else + { env->me_flags |= MDB_RDONLY; - else if ( fileTransferState == 0) - env->me_flags &= ~MDB_RDONLY; - else - ret = 1; // invalid parameter - - mdb_env_sync(env, 1); + } + } else if (op == MDB_STATE_GETXLOGNUM) + { + if (env->me_flags & MDB_WAL) + { + *last_xlog_num = env->me_walstate.xlog_num; + } + } else if ( op == MDB_STATE_CLEAR) + { + env->me_flags &= ~MDB_RDONLY; + env->me_flags &= ~MDB_KEEPXLOGS; } + else + ret = EINVAL; + + mdb_env_sync(env, 1); mdb_env_info(env, &env_stats); //dbSizeMb is used as a hint so that there is no need to transfer the bytes beyond this value. *dbSizeMb = 1 + (unsigned long)(((unsigned long long)env_stats.me_last_pgno * (unsigned long long)env->me_psize) >> 20); *dbMapSizeMb = (unsigned long)((unsigned long long)env->me_mapsize >> 20); if (db_path == NULL || db_path_size <= 0 || strlen(env->me_path) >= db_path_size) - ret = 3; + ret = EOVERFLOW; else strcpy(db_path, env->me_path); UNLOCK_MUTEX_W(env); return ret; } +/** @brief Set commit hook func for Raft + */ +void mdb_set_raft_prepare_commit_func(MDB_env *env, MDB_raft_prepare_commit_func *raft_prepare_commit_func) +{ + env->me_raft_prepare_commit_func = raft_prepare_commit_func; +} + +/** @brief callback for raft post commit - set raft volatle state with logIndex argument when commit succeeded + */ +void mdb_set_raft_post_commit_func(MDB_env *env, MDB_raft_post_commit_func *raft_post_commit_func) +{ + env->me_raft_post_commit_func = raft_post_commit_func; +} + +/** @brief callback for raft commit fail to write WAL or meta page (due to disk full/failure). The callback + * would set the server to Raft Follower role to avoid reusing the logIndex/logTerm for new client requests. + */ +void mdb_set_raft_commit_fail_func(MDB_env *env, MDB_raft_commit_fail_func *raft_commit_fail_func) +{ + env->me_raft_commit_fail_func = raft_commit_fail_func; +} + +#ifndef _WIN32 +/** @brief flush an array of dirty pages to WAL file + */ +static +int write_wal_pages(MDB_env *env, const struct iovec *iov, int n) +{ + int rc = 0; + int i = 0; + int nw = 0; + int total_bytes = 0; + char xlog_file[256]; + HANDLE fd = INVALID_HANDLE_VALUE; + + if (!(env->me_flags & MDB_WAL)) + goto done; + + if (env->me_walstate.xlog_fd == INVALID_HANDLE_VALUE) + { + sprintf(xlog_file, "%s/xlogs/%08lu", env->me_path, env->me_walstate.xlog_num); + fd = open(xlog_file, O_CREAT|O_WRONLY, S_IRUSR|S_IWUSR); + if (fd == INVALID_HANDLE_VALUE) + { + rc = ENOMEM; + goto fatal_error; + } + + env->me_walstate.xlog_fd = fd; + env->me_walstate.xlog_pages = 0; + env->me_walstate.txn_pages = 0; + } + + for(i=0; ime_walstate.xlog_pages + (total_bytes/env->me_psize)) >= (MDB_IDL_UM_MAX - 2)) + { + rc = ENOMEM; + goto fatal_error; + } + + nw = writev(env->me_walstate.xlog_fd, iov, n); + if (nw < 0) + { + rc = ENOMEM; + goto fatal_error; + } + + if (total_bytes != nw) + { + rc = ENOMEM; + goto fatal_error; + } + + env->me_walstate.txn_pages += nw/env->me_psize; + env->me_walstate.xlog_pages += nw/env->me_psize; + +done: + return rc; + +fatal_error: + //The server will shutdown, and go though WAL recovery procedure + // to cleanup imcomplete pages in WAL file. + env->me_flags|=MDB_FATAL_ERROR; + goto done; +} +#endif + unsigned long long mdb_env_get_lasttid(MDB_env *env) { @@ -8597,5 +9448,4 @@ mdb_env_get_lasttid(MDB_env *env) } return env->me_metas[0]->mm_txnid; } - /** @} */ diff --git a/vmdir/tools/vdcadmintool/ldapbindclient.c b/vmdir/tools/vdcadmintool/ldapbindclient.c index 50d6ae700..01dceedf8 100644 --- a/vmdir/tools/vdcadmintool/ldapbindclient.c +++ b/vmdir/tools/vdcadmintool/ldapbindclient.c @@ -253,7 +253,7 @@ _VdcadminClientTestSRPBind( DWORD dwError = 0; LDAP * pLD = NULL; - dwError = VmDirSASLSRPBind( &pLD, pszLDAPURI, pszDefaultBindUPN, pszDefaultPasswd ); + dwError = VmDirSASLSRPBindExt1( &pLD, pszLDAPURI, pszDefaultBindUPN, pszDefaultPasswd, MAX_LDAP_CONNECT_NETWORK_TIMEOUT ); BAIL_ON_VMDIR_ERROR(dwError); printf("%s SRP bind succeeded.\n\n", pszLDAPURI); diff --git a/vmdir/tools/vdcupgrade/main.c b/vmdir/tools/vdcupgrade/main.c index 7bb0a3170..a6721d6fa 100644 --- a/vmdir/tools/vdcupgrade/main.c +++ b/vmdir/tools/vdcupgrade/main.c @@ -243,7 +243,7 @@ VmDirMain( VmDirReadString("password: ", pszPasswordBuf, VMDIR_MAX_PWD_LEN+1, FALSE); } - dwError = VmDirSafeLDAPBind(&pLd, pszServerName, pszAdminUPN, pszPasswordBuf); + dwError = VmDirSafeLDAPBindExt1(&pLd, pszServerName, pszAdminUPN, pszPasswordBuf, MAX_LDAP_CONNECT_NETWORK_TIMEOUT); BAIL_ON_VMDIR_ERROR(dwError); dwError = getPSCVersion( diff --git a/vmdns/server/common/Makefile.am b/vmdns/server/common/Makefile.am index a969895d6..ba7bb4ca7 100755 --- a/vmdns/server/common/Makefile.am +++ b/vmdns/server/common/Makefile.am @@ -21,6 +21,7 @@ libsrvcommon_la_SOURCES = \ nameEntry.c \ recordlist.c \ recordobject.c \ + registry.c \ securityutils.c \ dnsprotocol.c \ serviceapi.c \ diff --git a/vmdns/server/common/defines.h b/vmdns/server/common/defines.h index 2a9a11ee4..735ebe6a5 100644 --- a/vmdns/server/common/defines.h +++ b/vmdns/server/common/defines.h @@ -85,6 +85,7 @@ extern "C" { #define VMAFD_CONFIG_PARAMETER_KEY_PATH "SYSTEM\\CurrentControlSet\\services\\VMWareAfdService\\Parameters" #define VMDNS_CONFIG_PARAMETER_KEY_PATH "SYSTEM\\CurrentControlSet\\services\\VMwareDNSService\\Parameters" #endif +#define VMDNS_MAX_CONFIG_VALUE_LENGTH 2048 typedef enum diff --git a/vmdns/server/common/includes.h b/vmdns/server/common/includes.h index 8e4a4ee36..39bb88b20 100755 --- a/vmdns/server/common/includes.h +++ b/vmdns/server/common/includes.h @@ -30,6 +30,9 @@ #include +#include +#include + #else #pragma once diff --git a/vmdns/server/common/prototypes.h b/vmdns/server/common/prototypes.h index c23d508ed..c7c8eb4f0 100755 --- a/vmdns/server/common/prototypes.h +++ b/vmdns/server/common/prototypes.h @@ -912,6 +912,19 @@ VmDnsSecIsRRTypeSec( DWORD dwRecordType ); +//registry.c +DWORD +VmDnsRegSaveForwarders( + DWORD dwCount, + PCSTR* ppszForwarders + ); + +DWORD +VmDnsRegLoadForwarders( + PDWORD pdwCount, + PSTR** pppszForwarders + ); + // Zone #ifdef __cplusplus } diff --git a/vmdns/server/common/registry.c b/vmdns/server/common/registry.c new file mode 100644 index 000000000..c33720917 --- /dev/null +++ b/vmdns/server/common/registry.c @@ -0,0 +1,370 @@ +/* + * Copyright © 2012-2015 VMware, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the “License”); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, without + * warranties or conditions of any kind, EITHER EXPRESS OR IMPLIED. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + + +#include "includes.h" + + +#define VMDNS_REG_VALUE_FORWARDER "Forwarders" + +static +DWORD +VmDnsArrayToString( + PCSTR* ppszStringArray, + DWORD dwCount, + PSTR* ppszString, + PDWORD pdwLengh + ); + + +static +DWORD +VmDnsStringToArray( + PCSTR ppszString, + DWORD dwStringLength, + PSTR** ppszStringArray, + PDWORD pdwCount + ); + +DWORD +VmDnsRegSaveForwarders( + DWORD dwCount, + PCSTR* ppszForwarders + ) +{ + DWORD dwError = 0; + HANDLE hConnection = NULL; + HKEY hRootKey = NULL; + HKEY hParamKey = NULL; + + PSTR pszStringForwarders = NULL; + DWORD dwStringForwardersLen = 0; + PCSTR pszParamsKeyPath = VMDNS_CONFIG_PARAMETER_KEY_PATH; + PCSTR pszForwarderValue = VMDNS_REG_VALUE_FORWARDER; + + + if (!ppszForwarders) + { + dwError = ERROR_INVALID_PARAMETER; + BAIL_ON_VMDNS_ERROR(dwError); + } + + dwError = VmDnsArrayToString( + ppszForwarders, + dwCount, + &pszStringForwarders, + &dwStringForwardersLen + ); + BAIL_ON_VMDNS_ERROR(dwError); + + dwError = RegOpenServer(&hConnection); + BAIL_ON_VMDNS_ERROR(dwError); + + dwError = RegOpenKeyExA( + hConnection, + NULL, + HKEY_THIS_MACHINE, + 0, + KEY_WRITE, + &hRootKey); + BAIL_ON_VMDNS_ERROR(dwError); + + dwError = RegOpenKeyExA( + hConnection, + hRootKey, + pszParamsKeyPath, + 0, + KEY_WRITE, + &hParamKey + ); + BAIL_ON_VMDNS_ERROR(dwError); + + dwError = RegSetValueExA( + hConnection, + hParamKey, + pszForwarderValue, + 0, + REG_MULTI_SZ, + (PVOID)pszStringForwarders, + dwStringForwardersLen + ); + BAIL_ON_VMDNS_ERROR(dwError); + +cleanup: + + + if (hParamKey) + { + RegCloseKey(hConnection, hParamKey); + } + if (hRootKey) + { + RegCloseKey(hConnection, hRootKey); + } + if (hConnection) + { + RegCloseServer(hConnection); + } + VMDNS_SAFE_FREE_MEMORY(pszStringForwarders); + + return dwError; +error: + + goto cleanup; +} + +DWORD +VmDnsRegLoadForwarders( + PDWORD pdwCount, + PSTR** pppszForwarders + ) +{ + DWORD dwError = 0; + HANDLE hConnection = NULL; + HKEY hRootKey = NULL; + HKEY hParamKey = NULL; + + DWORD dwCount = 0; + PSTR* ppszForwarders = NULL; + PCSTR pszParamsKeyPath = VMDNS_CONFIG_PARAMETER_KEY_PATH; + PCSTR pszForwarderValue = VMDNS_REG_VALUE_FORWARDER; + char szValue[VMDNS_MAX_CONFIG_VALUE_LENGTH] = {0}; + DWORD dwszValueSize = sizeof(szValue); + + + if (!pdwCount || !pppszForwarders) + { + dwError = ERROR_INVALID_PARAMETER; + BAIL_ON_VMDNS_ERROR(dwError); + } + + dwError = RegOpenServer(&hConnection); + BAIL_ON_VMDNS_ERROR(dwError); + + dwError = RegOpenKeyExA( + hConnection, + NULL, + HKEY_THIS_MACHINE, + 0, + KEY_READ, + &hRootKey); + BAIL_ON_VMDNS_ERROR(dwError); + + dwError = RegOpenKeyExA( + hConnection, + hRootKey, + pszParamsKeyPath, + 0, + KEY_READ, + &hParamKey + ); + BAIL_ON_VMDNS_ERROR(dwError); + + + dwError = RegGetValue( + hConnection, + hParamKey, + NULL, + pszForwarderValue, + 0, + NULL, + (PVOID*)&szValue, + &dwszValueSize + ); + + if (dwError == LWREG_ERROR_NO_SUCH_KEY_OR_VALUE) + { + dwError = ERROR_NO_DATA; + } + BAIL_ON_VMDNS_ERROR(dwError); + + dwError = VmDnsStringToArray( + szValue, + dwszValueSize, + &ppszForwarders, + &dwCount + ); + BAIL_ON_VMDNS_ERROR(dwError); + + *pppszForwarders = ppszForwarders; + *pdwCount = dwCount; + +cleanup: + + if (hParamKey) + { + RegCloseKey(hConnection, hParamKey); + } + if (hRootKey) + { + RegCloseKey(hConnection, hRootKey); + } + if (hConnection) + { + RegCloseServer(hConnection); + } + + return dwError; +error: + + if (pppszForwarders) + { + *pppszForwarders = NULL; + } + if (pdwCount) + { + *pdwCount = 0; + } + if (ppszForwarders) + { + VmDnsFreeStringArrayA(ppszForwarders); + } + + goto cleanup; +} + +static +DWORD +VmDnsArrayToString( + PCSTR* ppszStringArray, + DWORD dwCount, + PSTR* ppszString, + PDWORD pdwStringLength + ) +{ + DWORD dwError = 0; + PSTR pszString = NULL; + PSTR pszCursor = NULL; + DWORD dwStrLength = 0; + DWORD dwStrLengthUsed = 0; + DWORD dwIndex = 0; + + if (!ppszStringArray) + { + dwError = ERROR_INVALID_PARAMETER; + BAIL_ON_VMDNS_ERROR(dwError); + } + + for(; dwIndex < dwCount; ++dwIndex) + { + dwStrLength += (VmDnsStringLenA(ppszStringArray[dwIndex])) + 1; + } + dwStrLength++; + + dwError = VmDnsAllocateMemory(dwStrLength, (PVOID*)&pszString); + BAIL_ON_VMDNS_ERROR(dwError); + + pszCursor = pszString; + + for (dwIndex = 0; dwIndex < dwCount; ++dwIndex) + { + DWORD dwCurrStrLen = VmDnsStringLenA(ppszStringArray[dwIndex]); + dwError = VmDnsStringNCpyA( + pszCursor, + dwStrLength - dwStrLengthUsed, + ppszStringArray[dwIndex], + dwCurrStrLen + ); + BAIL_ON_VMDNS_ERROR(dwError); + + dwStrLengthUsed+= dwCurrStrLen+1; + pszCursor+= dwCurrStrLen+1; + } + + *ppszString = pszString; + *pdwStringLength = dwStrLength; +cleanup: + + return dwError; +error: + + if (ppszString) + { + *ppszString = NULL; + } + if (pdwStringLength) + { + *pdwStringLength = 0; + } + VMDNS_SAFE_FREE_MEMORY(pszString); + goto cleanup; +} + + +static +DWORD +VmDnsStringToArray( + PCSTR pszString, + DWORD dwStringLength, + PSTR** pppszStringArray, + PDWORD pdwCount + ) +{ + DWORD dwError = 0; + DWORD dwIndex = 0; + PCSTR pszCursor = pszString; + + PSTR* ppszStringArray = NULL; + DWORD dwCount = 0; + + while (pszCursor && VmDnsStringLenA(pszCursor)) + { + DWORD dwCursorLength = VmDnsStringLenA(pszCursor); + ++dwCount; + pszCursor += dwCursorLength+1; + } + + if (dwCount) + { + dwError = VmDnsAllocateMemory(sizeof(PSTR)*dwCount, (PVOID*)&ppszStringArray); + BAIL_ON_VMDNS_ERROR(dwError); + + pszCursor = pszString; + + for (;dwIndexpNext) { dwError = VmRESTRegisterHandler( - gpVdnsRESTHandle, + pHTTPHandle, pEndPoint->pszName, pHandlers, NULL @@ -82,7 +106,7 @@ VmDnsRESTServerInit( } } - dwError = VmRESTStart(gpVdnsRESTHandle); + dwError = VmRESTStart(pHTTPHandle); if (dwError) { // soft fail - will not listen on REST port. @@ -90,24 +114,51 @@ VmDnsRESTServerInit( dwError = 0; } + gpVdnsRESTHandle = pHTTPHandle; + cleanup: return dwError; error: + VmDnsFreeRESTHandle(pHTTPHandle); VmDnsLog(VMDNS_LOG_LEVEL_ERROR, "%s failed, error (%d)", __FUNCTION__, dwError); goto cleanup; } + VOID VmDnsRESTServerShutdown( VOID ) +{ + VmDnsFreeRESTHandle(gpVdnsRESTHandle); + gpVdnsRESTHandle = NULL; +} + +static +VOID +VmDnsFreeRESTHandle( + PVMREST_HANDLE pHandle + ) { PREST_API_MODULE pModule = NULL; + DWORD dwError = 0; - if (gpVdnsRESTHandle) + if (pHandle) { - VmRESTStop(gpVdnsRESTHandle); + /* + * REST library have detached threads, maximum time out specified is the max time + * allowed for the threads to finish their execution. + * If finished early, it will return success. + * If not able to finish in specified time, failure will be returned + */ + dwError = VmRESTStop(pHandle, VMDNS_REST_STOP_TIMEOUT_SEC); + + if (dwError != 0) + { + VmDnsLog(VMDNS_LOG_LEVEL_WARNING,"%s: rest stop error: %d",__FUNCTION__, dwError); + } + if (gpVdnsRestApiDef) { pModule = gpVdnsRestApiDef->pModules; @@ -117,11 +168,11 @@ VmDnsRESTServerShutdown( for (; pEndPoint; pEndPoint = pEndPoint->pNext) { (VOID)VmRESTUnRegisterHandler( - gpVdnsRESTHandle, pEndPoint->pszName); + pHandle, pEndPoint->pszName); } } } - VmRESTShutdown(gpVdnsRESTHandle); + VmRESTShutdown(pHandle); } } diff --git a/vmdns/server/rest-head/operation.c b/vmdns/server/rest-head/operation.c index 465d05df9..565b2f340 100755 --- a/vmdns/server/rest-head/operation.c +++ b/vmdns/server/rest-head/operation.c @@ -80,8 +80,8 @@ VmDnsRESTOperationReadRequest( dwError = VmRESTGetHttpMethod(pRestReq, &pRestOp->pszMethod); BAIL_ON_VMDNS_ERROR(dwError); - // read request URI - dwError = VmRESTGetHttpURI(pRestReq, &pRestOp->pszPath); + // read request URI. TRUE - Request c-rest-engine to decode URI + dwError = VmRESTGetHttpURI(pRestReq, TRUE, &pRestOp->pszPath); BAIL_ON_VMDNS_ERROR(dwError); pszTmp = VmDnsStringChrA(pRestOp->pszPath, '?'); diff --git a/vmdns/server/vmdns/globals.c b/vmdns/server/vmdns/globals.c index e704c010f..a6daa596d 100755 --- a/vmdns/server/vmdns/globals.c +++ b/vmdns/server/vmdns/globals.c @@ -51,5 +51,5 @@ VMDNS_GLOBALS gVmdnsGlobals = VMDNS_SF_INIT(.bEnableDNSProtocol, FALSE), //rest-head - VMDNS_SF_INIT(.pszRestListenPort, NULL) + VMDNS_SF_INIT(.dwRestListenPort, 0) }; diff --git a/vmdns/server/vmdns/shutdown.c b/vmdns/server/vmdns/shutdown.c index 658d38718..0f1f36f39 100755 --- a/vmdns/server/vmdns/shutdown.c +++ b/vmdns/server/vmdns/shutdown.c @@ -53,5 +53,4 @@ VmDnsCleanupGlobals( { VMDNS_SAFE_FREE_MEMORY(gVmdnsGlobals.pszLogFile); VmDnsFreeMutex( gVmdnsGlobals.pMutex); - VMDNS_SAFE_FREE_MEMORY(gVmdnsGlobals.pszRestListenPort); } diff --git a/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/IdentityManager.java b/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/IdentityManager.java index 59e8ddbde..b589be750 100644 --- a/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/IdentityManager.java +++ b/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/IdentityManager.java @@ -335,6 +335,8 @@ private ProvidersInfo(Collection providers, public static final String WELLKNOWN_SOLUTIONUSERS_GROUP_DESCRIPTION = "Well-known solution users' group, which contains all solution users as members."; public static final String WELLKNOWN_CONFIGURATIONUSERS_GROUP_NAME = "SystemConfiguration.Administrators"; public static final String WELLKNOWN_CONFIGURATIONUSERS_GROUP_DESCRIPTION = "Well-known configuration users' group which contains all configuration users as members."; + public static final String WELLKNOWN_TENANT_OPERATORS_GROUP_NAME = "TenantOperators"; + public static final String WELLKNOWN_TENANT_OPERATORS_GROUP_DESCRIPTION = "Well-known tenant operators group which contains all users with tenant operation privileges."; public static final String WELLKNOWN_ACT_AS_USERS_GROUP_NAME = "ActAsUsers"; public static final String WELLKNOWN_ACT_AS_USERS_GROUP_DESCRIPTION = "Well-known act-as users' group which contains all solution users that are allowed to act on behalf of person users."; public static final String WELLKNOWN_TRUSTED_USERS_GROUP_NAME = "TrustedUsers"; @@ -387,6 +389,8 @@ protected IdentityManager() throws IDMException { ensureValidTenant(systemTenant); ensureWellKnownGroupExists(systemTenant, WELLKNOWN_CONFIGURATIONUSERS_GROUP_NAME, WELLKNOWN_CONFIGURATIONUSERS_GROUP_DESCRIPTION); + ensureWellKnownGroupExists(systemTenant, WELLKNOWN_TENANT_OPERATORS_GROUP_NAME, + WELLKNOWN_TENANT_OPERATORS_GROUP_DESCRIPTION); // Start the Tenant Cache thread Thread idmCacheThread = new IdmCachePeriodicChecker(); diff --git a/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/provider/LdapConnectionPool.java b/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/provider/LdapConnectionPool.java index 36effec8d..5490aef03 100644 --- a/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/provider/LdapConnectionPool.java +++ b/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/provider/LdapConnectionPool.java @@ -56,7 +56,13 @@ public ILdapConnectionEx borrowConnection(PooledLdapConnectionIdentity identity) } public void returnConnection(PooledLdapConnection pooledConnection) { - if (pooledConnection == null || pooledConnection.getIdentity() == null || pooledConnection.getConnection() == null) { + if (pooledConnection == null || pooledConnection.getConnection() == null) { + return; + } + + if (pooledConnection.getIdentity() == null) { + logger.warn("Identity is not set. Closing connection"); + pooledConnection.getConnection().close(); return; } diff --git a/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/provider/activedirectory/ActiveDirectoryProvider.java b/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/provider/activedirectory/ActiveDirectoryProvider.java index 5604971c8..3a0a1109f 100644 --- a/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/provider/activedirectory/ActiveDirectoryProvider.java +++ b/vmidentity/idm/server/src/main/java/com/vmware/identity/idm/server/provider/activedirectory/ActiveDirectoryProvider.java @@ -1880,6 +1880,7 @@ private PooledLdapConnection borrowLdapConnection(String domainName, boolean bUs ? obtainDcInfoWithRediscover(domainName) : obtainDcInfo(domainName); String connStr; + PooledLdapConnectionIdentity identity = null; if (dcInfo != null) { if (!ServerUtils.isNullOrEmpty(dcInfo.domainFQDN)) @@ -1897,12 +1898,12 @@ private PooledLdapConnection borrowLdapConnection(String domainName, boolean bUs builder.setPassword(this.getStoreDataEx().getPassword()); builder.setUseGCPort(bUseGC); builder.setTenantName(tenantName); - PooledLdapConnectionIdentity identity = builder.build(); + identity = builder.build(); conn = pool.borrowConnection(identity); } - return new PooledLdapConnection(conn, null, pool); + return new PooledLdapConnection(conn, identity, pool); } // Get a GC ldap connection to the registered AD provider domain diff --git a/vmidentity/idm/server/src/test/java/com/vmware/identity/idm/server/LdapConnectionPoolTest.java b/vmidentity/idm/server/src/test/java/com/vmware/identity/idm/server/LdapConnectionPoolTest.java index ad7755729..264670501 100644 --- a/vmidentity/idm/server/src/test/java/com/vmware/identity/idm/server/LdapConnectionPoolTest.java +++ b/vmidentity/idm/server/src/test/java/com/vmware/identity/idm/server/LdapConnectionPoolTest.java @@ -18,7 +18,14 @@ import com.vmware.identity.idm.server.provider.PooledLdapConnection; import com.vmware.identity.idm.server.provider.PooledLdapConnectionIdentity; import com.vmware.identity.interop.ldap.ILdapConnectionEx; +import com.vmware.identity.interop.ldap.ILdapMessage; +import com.vmware.identity.interop.ldap.ILdapPagedSearchResult; +import com.vmware.identity.interop.ldap.LdapBindMethod; +import com.vmware.identity.interop.ldap.LdapControlNative; +import com.vmware.identity.interop.ldap.LdapMod; +import com.vmware.identity.interop.ldap.LdapOption; import com.vmware.identity.interop.ldap.LdapScope; +import com.vmware.identity.interop.ldap.TimevalNative; public class LdapConnectionPoolTest { @@ -163,6 +170,16 @@ public void test_returnConnection() throws Exception { pool.returnConnection(new PooledLdapConnection(conn, identity, pool)); } + @Test + public void test_returnConnection_NullIdentity() { + LdapConnectionPool pool = LdapConnectionPool.getInstance(); + pool.createPool(newTenantName); + LdapConnectionExTest conn = new LdapConnectionExTest(); + PooledLdapConnection pooledLdapConnectionWithNullIdentity = new PooledLdapConnection(conn, null, pool); + pool.returnConnection(pooledLdapConnectionWithNullIdentity); + Assert.assertTrue(conn.isClosed()); + } + @Test public void test_returnConnection_BuildNewIdentity() throws Exception { @@ -294,4 +311,127 @@ private static Collection getLdapsCertificate() { throw new IllegalStateException(e); } } + + private static class LdapConnectionExTest implements ILdapConnectionEx { + + private boolean isClosed = false; + + @Override + public void setOption(LdapOption option, int value) { + + } + + @Override + public void setOption(LdapOption option, boolean value) { + + } + + @Override + public void bindConnection(String dn, String cred, LdapBindMethod method) { + + } + + @Override + public void bindSaslConnection(String userName, String domainName, + String userPassword, String krbConfPath) { + + } + + @Override + public void addObject(String dn, LdapMod[] attributes) { + + } + + @Override + public void addObject(String dn, Collection attributes) { + + } + + @Override + public void modifyObject(String dn, LdapMod attribute) { + + } + + @Override + public void modifyObject(String dn, LdapMod[] attributes) { + + } + + @Override + public void modifyObject(String dn, Collection attributes) { + + } + + @Override + public ILdapMessage search(String base, LdapScope scope, String filter, + String[] attributes, boolean attributesOnly) { + return null; + } + + @Override + public ILdapMessage search(String base, LdapScope scope, String filter, + Collection attributes, boolean attributesOnly) { + return null; + } + + @Override + public ILdapMessage search_ext(String base, LdapScope scope, + String filter, Collection attributes, + boolean attributesOnly, Collection sctrls, + Collection cctrls, TimevalNative timeout, + int sizelimit) { + return null; + } + + @Override + public ILdapPagedSearchResult search_one_page(String base, + LdapScope scope, String filter, Collection attributes, + int pageSize, ILdapPagedSearchResult prevPagedSearchResult) { + return null; + } + + @Override + public Collection paged_search(String base, + LdapScope scope, String filter, Collection attributes, + int pageSize) { + return null; + } + + @Override + public void deleteObject(String dn) { + + } + + @Override + public void deleteObjectTree(String dn) { + + } + + @Override + public void close() { + this.isClosed = true; + } + + boolean isClosed() { + return this.isClosed; + } + + @Override + public void bindSaslSrpConnection(String upn, String userPassword) { + + } + + @Override + public void bindSaslConnection(String userName, String domainName, + String userPassword) { + + } + + @Override + public Collection paged_search(String base, + LdapScope scope, String filter, Collection attributes, + int pageSize, int limit) { + return null; + } + } } diff --git a/vmidentity/openidconnect/client/src/test/java/com/vmware/identity/openidconnect/client/OIDCClientITBase.java b/vmidentity/openidconnect/client/src/test/java/com/vmware/identity/openidconnect/client/OIDCClientITBase.java index 30a1e55eb..93bb2fa60 100644 --- a/vmidentity/openidconnect/client/src/test/java/com/vmware/identity/openidconnect/client/OIDCClientITBase.java +++ b/vmidentity/openidconnect/client/src/test/java/com/vmware/identity/openidconnect/client/OIDCClientITBase.java @@ -49,7 +49,6 @@ import com.vmware.identity.rest.idm.data.OIDCClientDTO; import com.vmware.identity.rest.idm.data.OIDCClientMetadataDTO; import com.vmware.identity.rest.idm.data.ResourceServerDTO; -import com.vmware.identity.rest.idm.data.TenantCredentialsDTO; import com.vmware.identity.rest.idm.data.TenantDTO; /** @@ -151,7 +150,7 @@ public static void setUp(String config) throws Exception { // create a non-system tenant createTenant(regularTenant, regularTenantAdminUsername, regularTenantAdminPassword, properties.getProperty("tenant1.issuer"), idmClientForSystemTenant); - Thread.sleep(10 * 1000); // wait for tenant creation to finish + Thread.sleep(15 * 1000); // wait for tenant creation to finish // retrieve OIDC meta data from regular tenant metadataHelper = new MetadataHelper.Builder(domainControllerFQDN) @@ -294,7 +293,7 @@ private static ClientID registerOidcClient(String tenant, String subjectDN, Stri @AfterClass public static void tearDown() throws Exception { - idmClientForRegularTenant.tenant().delete(regularTenant); + idmClientForSystemTenant.tenant().delete(regularTenant); vmdirClientForSystemTenant.user().delete(systemTenant, solutionUserName, systemTenant); vmdirClientForSystemTenant.user().delete(systemTenant, multiTenantSolutionUserName, systemTenant); idmClientForSystemTenant.oidcClient().delete(systemTenant, clientId.getValue()); diff --git a/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/SolutionUser.java b/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/SolutionUser.java index 778fa41f0..a3d8bb1fb 100644 --- a/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/SolutionUser.java +++ b/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/SolutionUser.java @@ -44,5 +44,5 @@ public RSAPublicKey getPublicKey() { return (RSAPublicKey) this.certificate.getPublicKey(); } - public boolean isMultuTenant() { return this.multiTenant; } + public boolean isMultiTenant() { return this.multiTenant; } } \ No newline at end of file diff --git a/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/TokenIssuer.java b/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/TokenIssuer.java index 594c7bc8f..fefca082c 100644 --- a/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/TokenIssuer.java +++ b/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/TokenIssuer.java @@ -219,7 +219,7 @@ private Subject subject() { } private boolean multiTenant() { - return (this.personUser == null) && (this.solutionUser != null) && (this.solutionUser.isMultuTenant()); + return (this.personUser == null) && (this.solutionUser != null) && (this.solutionUser.isMultiTenant()); } private List audience() { diff --git a/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/UserInfoRetriever.java b/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/UserInfoRetriever.java index 4be2010c8..39fba78df 100644 --- a/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/UserInfoRetriever.java +++ b/vmidentity/openidconnect/server/src/main/java/com/vmware/identity/openidconnect/server/UserInfoRetriever.java @@ -14,6 +14,7 @@ package com.vmware.identity.openidconnect.server; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -29,6 +30,7 @@ import com.vmware.identity.idm.IIdentityStoreData; import com.vmware.identity.idm.InvalidPrincipalException; import com.vmware.identity.idm.KnownSamlAttributes; +import com.vmware.identity.idm.ValidateUtil; import com.vmware.identity.idm.client.CasIdmClient; import com.vmware.identity.openidconnect.common.ErrorObject; import com.vmware.identity.openidconnect.common.Scope; @@ -59,28 +61,36 @@ public UserInfo retrieveUserInfo( String givenName = null; String familyName = null; + List groupMembership = null; + + Collection attributes = new ArrayList<>(); + if (user instanceof PersonUser) { - com.vmware.identity.idm.PersonUser idmPersonUser; - try { - idmPersonUser = this.idmClient.findPersonUser(user.getTenant(), user.getPrincipalId()); - } catch (Exception e) { - throw new ServerException(ErrorObject.serverError("idm error while retrieving person user"), e); - } - if (idmPersonUser == null) { - throw new ServerException(ErrorObject.invalidRequest("person user with specified id not found")); - } - givenName = idmPersonUser.getDetail().getFirstName(); - familyName = idmPersonUser.getDetail().getLastName(); + attributes.add(new Attribute(KnownSamlAttributes.ATTRIBUTE_USER_FIRST_NAME)); + attributes.add(new Attribute(KnownSamlAttributes.ATTRIBUTE_USER_LAST_NAME)); } - List groupMembership = null; if ( scope.contains(ScopeValue.ID_TOKEN_GROUPS) || scope.contains(ScopeValue.ID_TOKEN_GROUPS_FILTERED) || scope.contains(ScopeValue.ACCESS_TOKEN_GROUPS) || scope.contains(ScopeValue.ACCESS_TOKEN_GROUPS_FILTERED) || scope.contains(ScopeValue.RESOURCE_SERVER_ADMIN_SERVER)) { - groupMembership = computeGroupMembership(user); + attributes.add(new Attribute(KnownSamlAttributes.ATTRIBUTE_USER_GROUPS)); + } + + + Collection attributeValuePairs = getAttributes(user, attributes); + + for (AttributeValuePair entry : attributeValuePairs) { + String attributeName = entry.getAttrDefinition().getName(); + if (attributeName.equals(KnownSamlAttributes.ATTRIBUTE_USER_FIRST_NAME)) { + givenName = entry.getValues().isEmpty() ? null : entry.getValues().get(0); + } else if (attributeName.equals(KnownSamlAttributes.ATTRIBUTE_USER_LAST_NAME)) { + familyName = entry.getValues().isEmpty() ? null : entry.getValues().get(0); + } else if (attributeName.equals(KnownSamlAttributes.ATTRIBUTE_USER_GROUPS)) { + groupMembership = entry.getValues(); + } } String adminServerRole = null; @@ -151,19 +161,22 @@ private Set computeGroupMembershipFiltered(List groupMembership, return result; } - private List computeGroupMembership(User user) throws ServerException { - Collection attributeValuePairs; + private Collection getAttributes(User user, Collection attributes) throws ServerException { + if (attributes == null || attributes.isEmpty()) { + return Collections.emptyList(); + } + + ValidateUtil.validateNotNull(user, "user"); try { - attributeValuePairs = this.idmClient.getAttributeValues( + Collection attributeValuePairs = this.idmClient.getAttributeValues( user.getTenant(), user.getPrincipalId(), - Collections.singleton(new Attribute(KnownSamlAttributes.ATTRIBUTE_USER_GROUPS))); + attributes); + ValidateUtil.validateNotNull(attributeValuePairs, "attribute value pairs"); + return attributeValuePairs; } catch (Exception e) { - throw new ServerException(ErrorObject.serverError("idm error while retrieving group membership"), e); + throw new ServerException(ErrorObject.serverError("idm error while retrieving user attributes"), e); } - assert attributeValuePairs != null && attributeValuePairs.size() == 1; - AttributeValuePair attributeValuePair = attributeValuePairs.iterator().next(); - return attributeValuePair.getValues(); } private String computeAdminServerRole(User user, List groupMembership) throws ServerException { @@ -177,6 +190,8 @@ private String computeAdminServerRole(User user, List groupMembership) t role = "Administrator"; } else if (groupMembershipLowerCase.contains(systemTenantGroupNamePrefix + "systemconfiguration.administrators")) { role = "ConfigurationUser"; + } else if (groupMembershipLowerCase.contains(systemTenantGroupNamePrefix + "tenantoperators")) { + role = "TenantOperator"; } else if (groupMembershipLowerCase.contains(groupNamePrefix + "trustedusers")) { role = "TrustedUser"; } else if (groupMembershipLowerCase.contains(groupNamePrefix + "users")) { diff --git a/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/Role.java b/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/Role.java index 1fca5f108..c54b3c2f2 100644 --- a/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/Role.java +++ b/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/Role.java @@ -38,6 +38,13 @@ public enum Role { */ TRUSTED_USER("TrustedUser"), + /** + * A tenant operator with slightly elevated permissions. + * + *

    This role should only be valid on the system tenant. + */ + TENANT_OPERATOR("TenantOperator"), + /** * A configuration user with slightly elevated permissions. * @@ -46,7 +53,7 @@ public enum Role { CONFIGURATION_USER("ConfigurationUser"), /** - * An administrative user with full permissions. + * An administrative user with full permissions on the tenant. */ ADMINISTRATOR("Administrator"); @@ -97,4 +104,15 @@ public boolean is(Role other) { return compareTo(other) >= 0; } + /** + * Checks if the role belongs to system domain. + * + * @returntrue if this role is in system domain + */ + public boolean isSystemTenantDomain() { + if (this == TENANT_OPERATOR || this == CONFIGURATION_USER) { + return true; + } + return false; + } } diff --git a/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/RoleMapper.java b/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/RoleMapper.java index f603e34ce..2e3eb8882 100644 --- a/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/RoleMapper.java +++ b/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/RoleMapper.java @@ -44,25 +44,27 @@ public String getSystemTenantDomain() { public RoleGroup getRoleGroup(Role role) { return getRoleGroup(role, false); } - public RoleGroup getRoleGroup(Role role, boolean multiTenanted) { + public RoleGroup getRoleGroup(Role role, boolean isSystemTenantDomain) { switch (role) { case ADMINISTRATOR: - return new RoleGroup(role, "Administrators", subjectDomain(multiTenanted)); + return new RoleGroup(role, "Administrators", subjectDomain(isSystemTenantDomain)); case CONFIGURATION_USER: return new RoleGroup(role, "SystemConfiguration.Administrators", systemTenantDomain); + case TENANT_OPERATOR: + return new RoleGroup(role, "TenantOperators", systemTenantDomain); case TRUSTED_USER: - return new RoleGroup(role, "TrustedUsers", subjectDomain(multiTenanted)); + return new RoleGroup(role, "TrustedUsers", subjectDomain(isSystemTenantDomain)); case REGULAR_USER: - return new RoleGroup(role, "Users", subjectDomain(multiTenanted)); + return new RoleGroup(role, "Users", subjectDomain(isSystemTenantDomain)); case GUEST_USER: - return new RoleGroup(role, "Everyone", subjectDomain(multiTenanted)); + return new RoleGroup(role, "Everyone", subjectDomain(isSystemTenantDomain)); default: throw new IllegalArgumentException("Invalid Role: '" + role.toString() + "'"); } } - private String subjectDomain(boolean multiTenant) + private String subjectDomain(boolean isSystemTenantDomain) { - return multiTenant ? systemTenantDomain : tenantDomain; + return isSystemTenantDomain ? systemTenantDomain : tenantDomain; } } diff --git a/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/filter/AuthorizationRequestFilter.java b/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/filter/AuthorizationRequestFilter.java index 9c2f8dac6..f810d9733 100644 --- a/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/filter/AuthorizationRequestFilter.java +++ b/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/filter/AuthorizationRequestFilter.java @@ -83,7 +83,7 @@ private void filterByRole(ContainerRequestContext context, Role requiredRole) { StringManager sm = StringManager.getManager(Config.LOCALIZATION_PACKAGE_NAME); try { - ResourceAccessRequest request = ResourceAccessRequest.fromRequestContext(context); + ResourceAccessRequest request = ResourceAccessRequest.fromRequestContext(context, requiredRole); if (request == null) { diff --git a/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/request/ResourceAccessRequest.java b/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/request/ResourceAccessRequest.java index 3ce67bec2..8fad2a262 100644 --- a/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/request/ResourceAccessRequest.java +++ b/vmidentity/rest/core/server/src/main/java/com/vmware/identity/rest/core/server/authorization/request/ResourceAccessRequest.java @@ -47,6 +47,7 @@ import com.vmware.identity.rest.core.server.exception.ServerException; import com.vmware.identity.rest.core.server.util.ClientFactory; import com.vmware.identity.rest.core.server.util.PathParameters; +import com.vmware.identity.rest.core.server.util.PrincipalUtil; import com.vmware.identity.rest.core.util.StringManager; /** @@ -161,6 +162,16 @@ public void validateContents() throws InvalidTokenException { * @param requiredRole the role to ensure that the token is valid for */ public void validateRole(Role requiredRole) throws InsufficientRoleException { + // if the required role is in system tenant domain but the user is not in system tenant domain, + // the user should not have the permission + if (requiredRole.isSystemTenantDomain()) { + String userDomain = PrincipalUtil.fromName(token.getSubject()).getDomain(); + String systemTenantDomain = roleMapper.getSystemTenantDomain(); + if (!userDomain.equalsIgnoreCase(systemTenantDomain)) { + throw new InsufficientRoleException(sm.getString("auth.ise.wrong.role", requiredRole)); + } + } + if (token.getRole() != null) { checkRoleField(requiredRole); } else { @@ -188,7 +199,8 @@ private void checkGroupsField(Role requiredRole) throws InsufficientRoleExceptio Role role = roles[i]; while (role.is(requiredRole)) { - if (checkIfGroupExists(groupList, roleMapper.getRoleGroup(role, token.isSubjectMultiTenant()).getGroupNetbios())) { + boolean isSystemTenantDomain = requiredRole.isSystemTenantDomain() || token.isSubjectMultiTenant(); + if (checkIfGroupExists(groupList, roleMapper.getRoleGroup(role, isSystemTenantDomain).getGroupNetbios())) { return; } role = roles[--i]; @@ -212,9 +224,10 @@ private boolean checkIfGroupExists(List groupList, String group) { * Construct a ResourceAccessRequest from a {@link ContainerRequestContext}. * * @param context the request to parse from + * @param requiredRole the requested role for the request * @return the request object constructed from the specified context or null if no token exists */ - public static ResourceAccessRequest fromRequestContext(ContainerRequestContext context) throws ServerException { + public static ResourceAccessRequest fromRequestContext(ContainerRequestContext context, Role requiredRole) throws ServerException { boolean secure = isRequestSecure(context); CasIdmClient client = ClientFactory.getClient(); @@ -230,7 +243,7 @@ public static ResourceAccessRequest fromRequestContext(ContainerRequestContext c String tenant = getTenant(context, client); long skew = getSkew(tenant, client); - Certificate cert = getSigningCert(tenant, client); + Certificate cert = getSigningCert(tenant, client, requiredRole.isSystemTenantDomain()); AccessTokenVerifier verifier = getAccessTokenVerifier(context, info, skew, cert); @@ -385,11 +398,13 @@ private static long getSkew(String tenant, CasIdmClient client) throws ServerExc * * @param tenant the tenant to get the signing certificates for * @param client the IDM client to communicate with + * @param isSystemdomain whether to get signing cert from system tenant * @return the signing certificate for tenant * @throws ServerException if there is a server error preventing the retrieval of the certificate */ - private static Certificate getSigningCert(String tenant, CasIdmClient client) throws ServerException { + private static Certificate getSigningCert(String tenant, CasIdmClient client, boolean isSystemdomain) throws ServerException { List certList; + tenant = isSystemdomain ? getSystemTenant(client) : tenant; try { certList = client.getTenantCertificate(tenant); } catch (Exception e) { diff --git a/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/IntegrationTestBase.java b/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/IntegrationTestBase.java index ef4b37cc1..bc6f06e50 100644 --- a/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/IntegrationTestBase.java +++ b/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/IntegrationTestBase.java @@ -21,6 +21,8 @@ import org.apache.http.HttpException; import org.apache.http.client.ClientProtocolException; +import com.vmware.directory.rest.client.VmdirClient; +import com.vmware.identity.rest.core.client.UPNUtil; import com.vmware.identity.rest.core.client.exceptions.ClientException; import com.vmware.identity.rest.core.client.test.integration.IntegrationTestProperties; import com.vmware.identity.rest.idm.client.IdmClient; @@ -34,6 +36,7 @@ public class IntegrationTestBase { protected static TenantDTO testTenant; protected static IdmClient systemAdminClient; + protected static VmdirClient systemAdminVmdirClient; protected static IdmClient testAdminClient; public static void init(boolean withTestTenant) throws IOException, GeneralSecurityException, ClientException, HttpException { @@ -45,6 +48,12 @@ public static void init(boolean withTestTenant) throws IOException, GeneralSecur properties.getSystemDomain(), properties.getSystemAdminPassword()); + systemAdminVmdirClient = TestClientFactory.createVmdirClient(properties.getHost(), + properties.getSystemTenant(), + UPNUtil.buildUPN(properties.getSystemAdminUsername(), + properties.getSystemDomain()), + properties.getSystemAdminPassword()); + if (withTestTenant) { testTenant = TestGenerator.generateTenant(); @@ -60,8 +69,8 @@ public static void init(boolean withTestTenant) throws IOException, GeneralSecur } public static void cleanup(boolean withTestTenant) throws ClientProtocolException, HttpException, ClientException, IOException { - if (withTestTenant && testAdminClient != null) { - testAdminClient.tenant().delete(testTenant.getName()); + if (withTestTenant && systemAdminClient != null) { + systemAdminClient.tenant().delete(testTenant.getName()); } } diff --git a/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/TenantResourceIT.java b/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/TenantResourceIT.java index 7e898c76c..dcd28aca5 100644 --- a/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/TenantResourceIT.java +++ b/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/TenantResourceIT.java @@ -19,28 +19,52 @@ import java.io.IOException; import java.security.GeneralSecurityException; +import java.util.Arrays; +import java.util.List; import org.apache.http.HttpException; import org.apache.http.client.ClientProtocolException; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import com.vmware.directory.rest.common.data.MemberType; +import com.vmware.directory.rest.common.data.UserDTO; +import com.vmware.identity.rest.core.client.UPNUtil; import com.vmware.identity.rest.core.client.exceptions.ClientException; +import com.vmware.identity.rest.core.client.exceptions.client.ForbiddenException; +import com.vmware.identity.rest.idm.client.IdmClient; +import com.vmware.identity.rest.idm.client.test.integration.util.TestClientFactory; +import com.vmware.identity.rest.idm.client.test.integration.util.TestGenerator; +import com.vmware.identity.rest.idm.client.test.integration.util.UserGenerator; import com.vmware.identity.rest.idm.data.LockoutPolicyDTO; import com.vmware.identity.rest.idm.data.TenantConfigurationDTO; import com.vmware.identity.rest.idm.data.TenantDTO; public class TenantResourceIT extends IntegrationTestBase { + private static UserDTO testUserWithPriviledge; + private static UserDTO testUserWithoutPriviledge; + private static String systemTenant; + private static String testTenantName1 = TenantResourceIT.class.getSimpleName() + ".tenant1"; + private static String testTenantName2 = TenantResourceIT.class.getSimpleName() + ".tenant2"; + @BeforeClass public static void init() throws HttpException, IOException, GeneralSecurityException, ClientException { IntegrationTestBase.init(true); + systemTenant = properties.getSystemTenant(); + testUserWithPriviledge = systemAdminVmdirClient.user().create(systemTenant, TestGenerator.generateVmdirUser("testUser1", systemTenant, "Test user with tenant operator priviledge.")); + List members = Arrays.asList(UPNUtil.buildUPN(testUserWithPriviledge.getName() ,testUserWithPriviledge.getDomain())); + systemAdminVmdirClient.group().addMembers(systemTenant, "TenantOperators", systemTenant, members, MemberType.USER); + testUserWithoutPriviledge = systemAdminVmdirClient.user().create(systemTenant, TestGenerator.generateVmdirUser("testUser2", systemTenant, "Test user without tenant operator priviledge.")); } @AfterClass public static void cleanup() throws ClientProtocolException, HttpException, ClientException, IOException { IntegrationTestBase.cleanup(true); + systemAdminVmdirClient.user().delete(systemTenant, testUserWithPriviledge.getName(), testUserWithPriviledge.getDomain()); + systemAdminVmdirClient.user().delete(systemTenant, testUserWithoutPriviledge.getName(), testUserWithoutPriviledge.getDomain()); } @Test @@ -83,4 +107,35 @@ public void testUpdateConfig() throws ClientProtocolException, HttpException, Cl assertEquals(lockout.getMaxFailedAttempts(), actual.getLockoutPolicy().getMaxFailedAttempts()); } + @Test + public void testCreateAndDelete() throws Exception { + IdmClient idmClientWithPrviledge = TestClientFactory.createClient(properties.getHost(), + systemTenant, + UPNUtil.buildUPN(testUserWithPriviledge.getName() ,testUserWithPriviledge.getDomain()), + UserGenerator.PASSWORD); + idmClientWithPrviledge.tenant().create(TestGenerator.generateTenant(testTenantName1)); + Thread.sleep(15 * 1000); + + IdmClient idmClientWithoutPrviledge = TestClientFactory.createClient(properties.getHost(), + systemTenant, + UPNUtil.buildUPN(testUserWithoutPriviledge.getName(), testUserWithoutPriviledge.getDomain()), + UserGenerator.PASSWORD); + Exception ex = null; + try { + idmClientWithoutPrviledge.tenant().create(TestGenerator.generateTenant(testTenantName2)); + } catch (Exception e) { + ex = e; + } + Assert.assertTrue(ex instanceof ForbiddenException); + + ex = null; + try { + idmClientWithoutPrviledge.tenant().delete(testTenantName1); + } catch (Exception e) { + ex = e; + } + Assert.assertTrue(ex instanceof ForbiddenException); + + idmClientWithPrviledge.tenant().delete(testTenantName1); + } } diff --git a/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/util/TestGenerator.java b/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/util/TestGenerator.java index c498db3e1..5fa7d5aca 100644 --- a/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/util/TestGenerator.java +++ b/vmidentity/rest/idm/client/src/integration-test/java/com/vmware/identity/rest/idm/client/test/integration/util/TestGenerator.java @@ -57,9 +57,9 @@ public static KeyPair getKeyPair() { return keyPair; } - public static TenantDTO generateTenant() throws IOException, GeneralSecurityException { + public static TenantDTO generateTenant(String tenant) throws IOException, GeneralSecurityException { TenantDTO.Builder builder = new TenantDTO.Builder() - .withName(TEST_TENANT_NAME) + .withName(tenant) .withLongName(TEST_TENANT_LONG_NAME) .withKey(TEST_TENANT_KEY) .withCredentials(generateTenantCredentials()) @@ -69,6 +69,9 @@ public static TenantDTO generateTenant() throws IOException, GeneralSecurityExce return builder.build(); } + public static TenantDTO generateTenant() throws IOException, GeneralSecurityException { + return generateTenant(TEST_TENANT_NAME); + } public static ExternalIDPDTO generateExternalIDP(CertificateDTO certificate) throws GeneralSecurityException, IOException { return ExternalIdpGenerator.generateExternalIDP(certificate); } diff --git a/vmidentity/rest/idm/client/src/main/java/com/vmware/identity/rest/idm/client/TenantResource.java b/vmidentity/rest/idm/client/src/main/java/com/vmware/identity/rest/idm/client/TenantResource.java index 6b47cd4c2..cd3127628 100644 --- a/vmidentity/rest/idm/client/src/main/java/com/vmware/identity/rest/idm/client/TenantResource.java +++ b/vmidentity/rest/idm/client/src/main/java/com/vmware/identity/rest/idm/client/TenantResource.java @@ -60,7 +60,7 @@ public TenantResource(BaseClient parent) { /** * Create a new tenant. * - *

    Required Role: {@code system administrator}. + *

    Required Role: {@code tenant operator}. * * @param tenant the new tenant to be created. * @return the newly created tenant. @@ -100,7 +100,7 @@ public TenantDTO get(String tenant) throws ClientException, ClientProtocolExcept /** * Delete a tenant. * - *

    Required Role: {@code administrator}. + *

    Required Role: {@code tenant operator}. * * @param tenant the name of the tenant to delete. * @throws ClientException if a client side error occurs. diff --git a/vmidentity/rest/idm/server/src/main/java/com/vmware/identity/rest/idm/server/resources/TenantResource.java b/vmidentity/rest/idm/server/src/main/java/com/vmware/identity/rest/idm/server/resources/TenantResource.java index 5e6117997..258a9ba57 100644 --- a/vmidentity/rest/idm/server/src/main/java/com/vmware/identity/rest/idm/server/resources/TenantResource.java +++ b/vmidentity/rest/idm/server/src/main/java/com/vmware/identity/rest/idm/server/resources/TenantResource.java @@ -44,9 +44,7 @@ import com.vmware.identity.idm.IIdentityStoreData; import com.vmware.identity.idm.InvalidArgumentException; import com.vmware.identity.idm.InvalidPasswordPolicyException; -import com.vmware.identity.idm.LockoutPolicy; import com.vmware.identity.idm.NoSuchTenantException; -import com.vmware.identity.idm.PasswordPolicy; import com.vmware.identity.idm.PersonUser; import com.vmware.identity.idm.PrincipalId; import com.vmware.identity.idm.SearchCriteria; @@ -61,7 +59,6 @@ import com.vmware.identity.rest.core.server.exception.server.InternalServerErrorException; import com.vmware.identity.rest.core.server.resources.BaseResource; import com.vmware.identity.rest.core.server.util.PrincipalUtil; -import com.vmware.identity.rest.core.server.util.Validate; import com.vmware.identity.rest.idm.data.AuthenticationPolicyDTO; import com.vmware.identity.rest.idm.data.BrandPolicyDTO; import com.vmware.identity.rest.idm.data.GroupDTO; @@ -117,7 +114,7 @@ public TenantResource(@Context ContainerRequestContext request, @Context Securit */ @POST @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - @RequiresRole(role = Role.ADMINISTRATOR) + @RequiresRole(role = Role.TENANT_OPERATOR) public TenantDTO create(TenantDTO tenantDTO) { try { // Create tenant @@ -261,7 +258,7 @@ public SearchResultDTO searchMembers(@PathParam(PathParameters.TENANT_NAME) Stri * HTTP 500 {@link InternalServerErrorException} Otherwise */ @DELETE @Path(PathParameters.TENANT_NAME_VAR) - @RequiresRole(role = Role.ADMINISTRATOR) + @RequiresRole(role = Role.TENANT_OPERATOR) public void delete(@PathParam(PathParameters.TENANT_NAME) String tenantName) { try { getIDMClient().deleteTenant(tenantName); diff --git a/vmidentity/rest/idm/server/src/test/java/com/vmware/identity/rest/idm/server/test/resources/TenantResourceTest.java b/vmidentity/rest/idm/server/src/test/java/com/vmware/identity/rest/idm/server/test/resources/TenantResourceTest.java index b36554e06..d736d08de 100644 --- a/vmidentity/rest/idm/server/src/test/java/com/vmware/identity/rest/idm/server/test/resources/TenantResourceTest.java +++ b/vmidentity/rest/idm/server/src/test/java/com/vmware/identity/rest/idm/server/test/resources/TenantResourceTest.java @@ -36,8 +36,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Iterator; -import java.util.List; import java.util.Locale; import java.util.Set; @@ -82,8 +80,6 @@ import com.vmware.identity.rest.idm.data.attributes.SearchType; import com.vmware.identity.rest.idm.data.attributes.TenantConfigType; import com.vmware.identity.rest.idm.server.mapper.AuthenticationPolicyMapper; -import com.vmware.identity.rest.idm.server.mapper.LockoutPolicyMapper; -import com.vmware.identity.rest.idm.server.mapper.PasswordPolicyMapper; import com.vmware.identity.rest.idm.server.resources.CertificateResource; import com.vmware.identity.rest.idm.server.resources.GroupResource; import com.vmware.identity.rest.idm.server.resources.IdentityProviderResource; @@ -200,7 +196,7 @@ public void testCreateTenant() throws Exception { @Test(expected=BadRequestException.class) public void testCreateTenant_WithInvalidTenantCredentials() { TenantDTO tenantDTO = TenantDTO.builder() - .withName(TENANT_NAME) + .withUsername(TENANT_NAME) .withLongName(TENANT_LONG_NAME) .withKey(TENANT_KEY) .withCredentials(null) diff --git a/vmidentity/ssoclients/common/include/public/ssotypes.h b/vmidentity/ssoclients/common/include/public/ssotypes.h index a0135797e..26ea3ab75 100644 --- a/vmidentity/ssoclients/common/include/public/ssotypes.h +++ b/vmidentity/ssoclients/common/include/public/ssotypes.h @@ -14,8 +14,8 @@ #ifndef _SSO_TYPES_H_ #define _SSO_TYPES_H_ +#include -typedef unsigned char bool; #ifndef false #define false 0 #endif