Skip to content

Commit a8d3f4e

Browse files
Mohammed Dasfacebook-github-bot
Mohammed Das
authored and
facebook-github-bot
committed
B&R for UDP encryption binary (facebookresearch#2200)
Summary: Pull Request resolved: facebookresearch#2200 As title Differential Revision: D44179453 fbshipit-source-id: 36a50e8f3ef8911bfd383bb843c255d238a3b09c
1 parent 905c8ba commit a8d3f4e

File tree

8 files changed

+43
-182
lines changed

8 files changed

+43
-182
lines changed

‎binaries_out_lists/emp_games.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ pcf2_attribution_calculator
88
pcf2_aggregation_calculator
99
shard_aggregator
1010
private_id_dfca_aggregator
11+
udp_encryptor

‎docker/emp_games/CMakeLists.txt‎

Lines changed: 21 additions & 179 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,16 @@
33
# This source code is licensed under the MIT license found in the
44
# LICENSE file in the root directory of this source tree.
55

6-
cmake_minimum_required(VERSION 3.10.1)
6+
cmake_minimum_required(VERSION 3.16)
77
project(game)
88

99
set(CMAKE_CXX_STANDARD 20)
1010
set(CMAKE_CXX_STANDARD_REQUIRED ON)
11+
add_library(range_v3 INTERFACE IMPORTED)
12+
set_target_properties(range_v3 PROPERTIES
13+
INTERFACE_INCLUDE_DIRECTORIES /root/build/range-v3/include)
14+
find_program(CMAKE_CXX_COMPILER clang++)
15+
1116
# Don't compile with AVX512 instructions since many of the AWS
1217
# instances won't have access to that instruction set.
1318
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mno-avx512f -march=haswell")
@@ -16,188 +21,25 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mno-avx512f -march=haswell")
1621
include("common.cmake")
1722
include("perf_tools.cmake")
1823

19-
# lift
20-
add_executable(
21-
lift_calculator
22-
"fbpcs/emp_games/lift/calculator/main.cpp"
23-
"fbpcs/emp_games/lift/calculator/OutputMetrics.hpp"
24-
"fbpcs/emp_games/lift/common/GroupedLiftMetrics.h"
25-
"fbpcs/emp_games/lift/common/GroupedLiftMetrics.cpp"
26-
"fbpcs/emp_games/lift/common/LiftMetrics.h"
27-
"fbpcs/emp_games/lift/common/LiftMetrics.cpp"
28-
"fbpcs/emp_games/lift/calculator/CalculatorApp.h"
29-
"fbpcs/emp_games/lift/calculator/CalculatorApp.cpp"
30-
"fbpcs/emp_games/lift/calculator/CalculatorGame.h"
31-
"fbpcs/emp_games/lift/calculator/OutputMetrics.h"
32-
"fbpcs/emp_games/lift/calculator/InputData.cpp"
33-
"fbpcs/emp_games/lift/calculator/InputData.h"
34-
"fbpcs/emp_games/lift/calculator/CalculatorGameConfig.h"
35-
"fbpcs/emp_games/lift/calculator/OutputMetricsData.h"
36-
"fbpcs/emp_games/common/PrivateData.h"
37-
"fbpcs/emp_games/common/SecretSharing.h"
38-
"fbpcs/emp_games/common/EmpOperationUtil.h"
39-
"fbpcs/emp_games/common/Csv.h"
40-
"fbpcs/emp_games/common/Csv.cpp")
41-
target_link_libraries(
42-
lift_calculator
43-
empgamecommon)
44-
install(TARGETS lift_calculator DESTINATION bin)
45-
46-
# generic shard_aggregator
47-
file(GLOB shard_aggregator_src
48-
"fbpcs/emp_games/attribution/shard_aggregator/AggMetrics.cpp",
49-
"fbpcs/emp_games/attribution/shard_aggregator/AggMetricsThresholdCheckers.cpp",
50-
"fbpcs/emp_games/attribution/shard_aggregator/ShardAggregatorApp.cpp",
51-
"fbpcs/emp_games/attribution/shard_aggregator/ShardAggregatorValidation.cpp",
52-
"fbpcs/emp_games/attribution/shard_aggregator/main.cpp"
53-
"fbpcs/emp_games/attribution/Aggregator.h"
54-
"fbpcs/emp_games/attribution/AttributionMetrics.h"
55-
"fbpcs/emp_games/attribution/AttributionRule.h"
56-
"fbpcs/emp_games/attribution/Constants.h"
57-
"fbpcs/emp_games/attribution/Conversion.h"
58-
"fbpcs/emp_games/attribution/Debug.h"
59-
"fbpcs/emp_games/attribution/Touchpoint.h",
60-
"fbpcs/emp_games/attribution/shard_aggregator/MainUtil.h",
61-
"fbpcs/emp_games/attribution/shard_aggregator/ShardAggregatorGame.h")
62-
add_executable(
63-
shard_aggregator
64-
${shard_aggregator_src})
65-
target_link_libraries(
66-
shard_aggregator
67-
empgamecommon
68-
perftools)
69-
install(TARGETS shard_aggregator DESTINATION bin)
70-
71-
# decoupled_attribution
72-
file(GLOB decoupled_attribution_src
73-
"fbpcs/emp_games/attribution/decoupled_attribution/**.c"
74-
"fbpcs/emp_games/attribution/decoupled_attribution/**.cpp"
75-
"fbpcs/emp_games/attribution/decoupled_attribution/**.h"
76-
"fbpcs/emp_games/attribution/decoupled_attribution/**.hpp")
77-
list(FILTER decoupled_attribution_src EXCLUDE REGEX ".*Test.*")
78-
add_executable(
79-
decoupled_attribution_calculator
80-
${decoupled_attribution_src})
81-
target_link_libraries(
82-
decoupled_attribution_calculator
83-
empgamecommon
84-
perftools)
85-
install(TARGETS decoupled_attribution_calculator DESTINATION bin)
86-
87-
# decoupled_aggregation
88-
file(GLOB decoupled_aggregation_src
89-
"fbpcs/emp_games/attribution/decoupled_aggregation/**.c"
90-
"fbpcs/emp_games/attribution/decoupled_aggregation/**.cpp"
91-
"fbpcs/emp_games/attribution/decoupled_aggregation/**.h"
92-
"fbpcs/emp_games/attribution/decoupled_aggregation/metadata/**.h"
93-
"fbpcs/emp_games/attribution/decoupled_aggregation/**.hpp")
94-
list(FILTER decoupled_aggregation_src EXCLUDE REGEX ".*Test.*")
95-
add_executable(
96-
decoupled_aggregation_calculator
97-
${decoupled_aggregation_src})
98-
target_link_libraries(
99-
decoupled_aggregation_calculator
100-
empgamecommon
101-
perftools)
102-
install(TARGETS decoupled_aggregation_calculator DESTINATION bin)
103-
104-
105-
# Find boost
106-
find_package(Boost REQUIRED)
107-
include_directories(${Boost_INCLUDE_DIRS})
108-
109-
# pcf2_attribution
110-
file(GLOB pcf2_attribution_src
111-
"fbpcs/emp_games/pcf2_attribution/**.c"
112-
"fbpcs/emp_games/pcf2_attribution/**.cpp"
113-
"fbpcs/emp_games/pcf2_attribution/**.h"
114-
"fbpcs/emp_games/pcf2_attribution/**.hpp")
115-
list(FILTER pcf2_attribution_src EXCLUDE REGEX ".*Test.*")
116-
add_executable(
117-
pcf2_attribution_calculator
118-
${pcf2_attribution_src})
119-
target_link_libraries(
120-
pcf2_attribution_calculator
121-
empgamecommon
122-
perftools)
123-
install(TARGETS pcf2_attribution_calculator DESTINATION bin)
124-
125-
# pcf2_aggregation
126-
file(GLOB pcf2_aggregation_src
127-
"fbpcs/emp_games/pcf2_aggregation/**.c"
128-
"fbpcs/emp_games/pcf2_aggregation/**.cpp"
129-
"fbpcs/emp_games/pcf2_aggregation/**.h"
130-
"fbpcs/emp_games/pcf2_aggregation/**.hpp")
131-
list(FILTER pcf2_aggregation_src EXCLUDE REGEX ".*Test.*")
132-
add_executable(
133-
pcf2_aggregation_calculator
134-
${pcf2_aggregation_src})
135-
target_link_libraries(
136-
pcf2_aggregation_calculator
137-
empgamecommon
138-
perftools)
139-
install(TARGETS pcf2_aggregation_calculator DESTINATION bin)
24+
message(STATUS "CMAKE_CXX_STANDARD: ${CMAKE_CXX_STANDARD}")
25+
message(STATUS "CMAKE_CXX_COMPILER: ${CMAKE_CXX_COMPILER}")
26+
message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
27+
message(STATUS "CMAKE_LINKER: ${CMAKE_LINKER}")
14028

141-
# pcf2_lift metadata compaction
142-
file(GLOB pcf2_lift_metadata_compaction_src
143-
"fbpcs/emp_games/lift/metadata_compaction/**.cpp"
144-
"fbpcs/emp_games/lift/metadata_compaction/**.h")
145-
list(FILTER pcf2_lift_metadata_compaction_src EXCLUDE REGEX ".*Test.*")
29+
# pcf2_udp encryption
30+
file(GLOB udp_encryptor_src
31+
"fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor/**.cpp"
32+
"fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor/**.h")
33+
list(FILTER udp_encryptor_src EXCLUDE REGEX ".*Test.*")
14634
add_executable(
147-
pcf2_lift_metadata_compaction
148-
${pcf2_lift_metadata_compaction_src})
35+
udp_encryptor
36+
${udp_encryptor_src})
14937
target_link_libraries(
150-
pcf2_lift_metadata_compaction
38+
udp_encryptor
15139
empgamecommon
15240
perftools
153-
pcf2_lift_input_processing
41+
${FOLLY_LIBRARIES}
15442
${Boost_LIBRARIES}
43+
range_v3
15544
)
156-
install(TARGETS pcf2_lift_metadata_compaction DESTINATION bin)
157-
158-
# pcf2_lift
159-
file(GLOB pcf2_lift_calculator_src
160-
"fbpcs/emp_games/lift/common/**.cpp"
161-
"fbpcs/emp_games/lift/common/**.h"
162-
"fbpcs/emp_games/lift/pcf2_calculator/**.cpp"
163-
"fbpcs/emp_games/lift/pcf2_calculator/**.h")
164-
list(FILTER pcf2_lift_calculator_src EXCLUDE REGEX ".*Test.*")
165-
add_executable(
166-
pcf2_lift_calculator
167-
${pcf2_lift_calculator_src})
168-
target_link_libraries(
169-
pcf2_lift_calculator
170-
empgamecommon
171-
perftools
172-
pcf2_lift_input_processing)
173-
install(TARGETS pcf2_lift_calculator DESTINATION bin)
174-
175-
# pcf2_shard_combiner
176-
file(GLOB pcf2_shard_combiner_src
177-
"fbpcs/emp_games/pcf2_shard_combiner/**.cpp"
178-
"fbpcs/emp_games/pcf2_shard_combiner/**.h"
179-
"fbpcs/emp_games/pcf2_shard_combiner/util/**.h")
180-
list(FILTER pcf2_shard_combiner_src EXCLUDE REGEX ".*Test.*")
181-
add_executable(
182-
pcf2_shard_combiner
183-
${pcf2_shard_combiner_src})
184-
target_link_libraries(
185-
pcf2_shard_combiner
186-
empgamecommon
187-
perftools)
188-
install(TARGETS pcf2_shard_combiner DESTINATION bin)
189-
190-
# private_id_dfca_aggregator
191-
file(GLOB private_id_dfca_aggregator_src
192-
"fbpcs/emp_games/private_id_dfca_aggregator/**.cpp"
193-
"fbpcs/emp_games/private_id_dfca_aggregator/**.h"
194-
"fbpcs/emp_games/private_id_dfca_aggregator/util/**.h")
195-
list(FILTER private_id_dfca_aggregator_src EXCLUDE REGEX ".*Test.*")
196-
add_executable(
197-
private_id_dfca_aggregator
198-
${private_id_dfca_aggregator_src})
199-
target_link_libraries(
200-
private_id_dfca_aggregator
201-
empgamecommon
202-
perftools)
203-
install(TARGETS private_id_dfca_aggregator DESTINATION bin)
45+
install(TARGETS udp_encryptor DESTINATION bin)

‎docker/emp_games/Dockerfile.ubuntu‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ ARG fbpcf_image
77

88
FROM ${fbpcf_image} as dev
99

10+
WORKDIR /root/build
11+
RUN git clone https://github.com/ericniebler/range-v3
12+
13+
14+
1015
RUN mkdir -p /root/build/emp_game
1116
WORKDIR /root/build/emp_game
1217

@@ -25,6 +30,9 @@ COPY fbpcs/emp_games/pcf2_shard_combiner/ ./fbpcs/emp_games/pcf2_shard_combiner
2530
COPY fbpcs/emp_games/private_id_dfca_aggregator/ ./fbpcs/emp_games/private_id_dfca_aggregator
2631
COPY fbpcs/emp_games/lift/ ./fbpcs/emp_games/lift
2732
COPY fbpcs/emp_games/common/ ./fbpcs/emp_games/common
33+
COPY fbpcs/emp_games/data_processing/global_parameters ./fbpcs/emp_games/data_processing/global_parameters
34+
COPY fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor ./fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor
35+
COPY fbpcs/emp_games/data_processing/unified_data_process/UdpDecryptor ./fbpcs/emp_games/data_processing/unified_data_process/UdpDecryptor
2836

2937
RUN cmake . -DTHREADING=ON -DUSE_RANDOM_DEVICE=ON
3038
RUN ./make_and_install_binary.sh
@@ -45,7 +53,8 @@ RUN apt-get -y update && apt-get install -y --no-install-recommends \
4553
libgoogle-glog0v5 \
4654
libssl1.1 \
4755
libre2-5 \
48-
zlib1g
56+
zlib1g \
57+
librange-v3-dev
4958

5059
COPY --from=dev /root/build/emp_game/bin/. /usr/local/bin/.
5160
RUN useradd -ms /bin/bash pcs

‎docker/emp_games/common.cmake‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ file(GLOB emp_game_common_src
1818
"fbpcs/emp_games/common/**.c"
1919
"fbpcs/emp_games/common/**.cpp"
2020
"fbpcs/emp_games/common/**.h"
21-
"fbpcs/emp_games/common/**.hpp")
21+
"fbpcs/emp_games/common/**.hpp"
22+
"fbpcs/emp_games/data_processing/global_parameters/**.cpp"
23+
"fbpcs/emp_games/data_processing/global_parameters/**.h"
24+
)
2225
list(FILTER emp_game_common_src EXCLUDE REGEX ".*Test.*")
2326
add_library(empgamecommon STATIC
2427
${emp_game_common_src})
@@ -30,7 +33,7 @@ target_link_libraries(
3033
${AWSSDK_LINK_LIBRARIES}
3134
${EMP-OT_LIBRARIES}
3235
google-cloud-cpp::storage
33-
Folly::folly
36+
${FOLLY_LIBRARIES}
3437
re2)
3538

3639
# pcf2 lift input processing

‎extract-docker-binaries.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ docker create -ti --name temp_container "${DOCKER_IMAGE_PATH}"
6868
docker cp temp_container:/usr/local/bin/lift_calculator "$SCRIPT_DIR/binaries_out/."
6969
docker cp temp_container:/usr/local/bin/pcf2_lift_calculator "$SCRIPT_DIR/binaries_out/."
7070
docker cp temp_container:/usr/local/bin/pcf2_lift_metadata_compaction "$SCRIPT_DIR/binaries_out/."
71+
docker cp temp_container:/usr/local/bin/udp_encryptor "$SCRIPT_DIR/binaries_out/."
7172
docker cp temp_container:/usr/local/bin/decoupled_attribution_calculator "$SCRIPT_DIR/binaries_out/."
7273
docker cp temp_container:/usr/local/bin/decoupled_aggregation_calculator "$SCRIPT_DIR/binaries_out/."
7374
docker cp temp_container:/usr/local/bin/pcf2_attribution_calculator "$SCRIPT_DIR/binaries_out/."

‎promote_scripts/promote_binaries.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ binary_names=(
4242
'data_processing/private_id_dfca_id_combiner'
4343
'validation/pc_pre_validation_cli'
4444
'smart_agent/smart_agent_server'
45+
'udp_encryptor'
4546
)
4647

4748
s3_path="s3://one-docker-repository-prod"

‎upload-binaries-to-s3-test.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ data_processing_repo="s3://$one_docker_repo/data_processing"
4646
private_id_repo="s3://$one_docker_repo/pid"
4747
validation_repo="s3://$one_docker_repo/validation"
4848
smart_agent_repo="s3://$one_docker_repo/smart_agent"
49+
udp_encryptor_package="s3://$one_docker_repo/data_processing/unified_data_process/UdpEncryptor/${TAG}/udp_encryptor"
4950

5051
if [ "$PACKAGE" = "emp_games" ]; then
5152
cd binaries_out || exit
@@ -59,6 +60,7 @@ aws s3 cp pcf2_aggregation_calculator "$pcf2_aggregation"
5960
aws s3 cp shard_aggregator "$shard_aggregator_package"
6061
aws s3 cp pcf2_shard_combiner "$pcf2_shard_combiner_package"
6162
aws s3 cp private_id_dfca_aggregator "$private_id_dfca_aggregator_package"
63+
aws s3 cp udp_encryptor "$udp_encryptor_package"
6264
cd .. || exit
6365
fi
6466

‎upload_scripts/upload-binaries-using-onedocker.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ pcf2_aggregation_path="private_attribution/pcf2_aggregation"
6363
shard_aggregator_path="private_attribution/shard-aggregator"
6464
pcf2_shard_combiner_path="private_attribution/pcf2_shard-combiner"
6565
private_id_dfca_aggregator_path="private_id_dfca/private_id_dfca_aggregator"
66+
udp_encryptor_path="data_processing/unified_data_process/UdpEncryptor"
6667

6768
if [ "$PACKAGE" = "emp_games" ]; then
6869
cd binaries_out || exit
@@ -76,6 +77,7 @@ onedocker_upload "$pcf2_aggregation_path" pcf2_aggregation_calculator
7677
onedocker_upload "$shard_aggregator_path" shard_aggregator
7778
onedocker_upload "$pcf2_shard_combiner_path" pcf2_shard_combiner
7879
onedocker_upload "$private_id_dfca_aggregator_path" private_id_dfca_aggregator
80+
onedocker_upload "$udp_encryptor_path" udp_encryptor
7981
cd .. || exit
8082
fi
8183

0 commit comments

Comments
 (0)