Skip to content

Commit dcaa544

Browse files
GUIGUI
committed
Get tests passing against elasticsearch v6. Fix end-of-year issues.
Our date parsing and formatting with ICU was incorrectly using "YYYY" instead of "yyyy", which was leading it to use the week-based year, which could cause unexpected dates around the year boundaries.
1 parent 4bf1d1c commit dcaa544

16 files changed

Lines changed: 197 additions & 133 deletions

File tree

‎src/api-umbrella/utils/elasticsearch.lua‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
local http = require "resty.http"
2+
local is_empty = require("pl.types").is_empty
3+
local json_decode = require("cjson").decode
4+
local json_encode = require "api-umbrella.utils.json_encode"
5+
6+
local server = config["elasticsearch"]["_first_server"]
7+
8+
local _M = {}
9+
10+
function _M.query(path, options)
11+
local httpc = http.new()
12+
13+
if not options then
14+
options = {}
15+
end
16+
17+
options["path"] = path
18+
19+
if not options["headers"] then
20+
options["headers"] = {}
21+
end
22+
23+
if server["userinfo"] and not options["headers"]["Authorization"] then
24+
options["headers"]["Authorization"] = "Basic " .. ngx.encode_base64(server["userinfo"])
25+
end
26+
27+
if options["body"] and type(options["body"]) == "table" then
28+
options["body"] = json_encode(options["body"])
29+
30+
if not options["headers"]["Content-Type"] then
31+
options["headers"]["Content-Type"] = "application/json"
32+
end
33+
end
34+
35+
local connect_ok, connect_err = httpc:connect(server["host"], server["port"])
36+
if connect_err then
37+
httpc:close()
38+
return nil, "elasticsearch connect error: " .. (connect_err or "")
39+
end
40+
41+
local res, err = httpc:request(options)
42+
if err then
43+
httpc:close()
44+
return nil, "elasticsearch request error: " .. (err or "")
45+
end
46+
47+
local body, body_err = res:read_body()
48+
if body_err then
49+
httpc:close()
50+
return nil, "elasticsearch read body error: " .. (body_err or "")
51+
end
52+
53+
local keepalive_ok, keepalive_err = httpc:set_keepalive()
54+
if not keepalive_ok then
55+
httpc:close()
56+
return nil, "elasticsearch keepalive error: " .. (keepalive_err or "")
57+
end
58+
59+
if res.status >= 400 and res.status ~= 404 then
60+
return nil, "Unsuccessful response: " .. (body or "")
61+
else
62+
if res.headers["Content-Type"] and string.sub(res.headers["Content-Type"], 1, 16) == "application/json" and not is_empty(body) then
63+
res["body_json"] = json_decode(body)
64+
end
65+
66+
return res
67+
end
68+
end
69+
70+
return _M

‎test/admin_ui/test_legacy_redirects.rb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ def setup
99
super
1010
setup_server
1111

12-
ElasticsearchHelper.clean_es_indices(["2014-11", "2015-01", "2015-03"])
12+
LogItem.clean_indices!
1313
FactoryBot.create(:log_item, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc)
14-
LogItem.gateway.refresh_index!
14+
LogItem.refresh_indices!
1515
end
1616

1717
def test_drilldown

‎test/admin_ui/test_stats_drilldown.rb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,13 @@ class Test::AdminUi::TestStatsDrilldown < Minitest::Capybara::Test
99
def setup
1010
super
1111
setup_server
12-
ElasticsearchHelper.clean_es_indices(["2014-11", "2015-01", "2015-03"])
12+
LogItem.clean_indices!
1313
end
1414

1515
def test_csv_download
1616
FactoryBot.create_list(:log_item, 5, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc)
1717
FactoryBot.create_list(:log_item, 5, :request_at => 1421413588000)
18-
LogItem.gateway.refresh_index!
18+
LogItem.refresh_indices!
1919
default_query = JSON.generate({
2020
"condition" => "AND",
2121
"rules" => [{

‎test/admin_ui/test_stats_logs.rb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,12 @@ class Test::AdminUi::TestStatsLogs < Minitest::Capybara::Test
99
def setup
1010
super
1111
setup_server
12-
ElasticsearchHelper.clean_es_indices(["2014-11", "2015-01", "2015-03"])
12+
LogItem.clean_indices!
1313
end
1414

1515
def test_xss_escaping_in_table
1616
log = FactoryBot.create(:xss_log_item, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_method => "OPTIONS")
17-
LogItem.gateway.refresh_index!
17+
LogItem.refresh_indices!
1818

1919
admin_login
2020
visit "/admin/#/stats/logs?search=&start_at=2015-01-12&end_at=2015-01-18&interval=day"
@@ -33,7 +33,7 @@ def test_xss_escaping_in_table
3333

3434
def test_csv_download_link_changes_with_filters
3535
FactoryBot.create(:log_item, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc)
36-
LogItem.gateway.refresh_index!
36+
LogItem.refresh_indices!
3737
default_query = JSON.generate({
3838
"condition" => "AND",
3939
"rules" => [{
@@ -117,7 +117,7 @@ def test_csv_download_link_changes_with_filters
117117
def test_csv_download
118118
FactoryBot.create_list(:log_item, 5, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_method => "OPTIONS")
119119
FactoryBot.create_list(:log_item, 5, :request_at => 1421413588000, :request_method => "OPTIONS")
120-
LogItem.gateway.refresh_index!
120+
LogItem.refresh_indices!
121121

122122
admin_login
123123
visit "/admin/#/stats/logs?search=&start_at=2015-01-12&end_at=2015-01-18&interval=day"

‎test/admin_ui/test_stats_map.rb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,13 @@ class Test::AdminUi::TestStatsMap < Minitest::Capybara::Test
99
def setup
1010
super
1111
setup_server
12-
ElasticsearchHelper.clean_es_indices(["2014-11", "2015-01", "2015-03"])
12+
LogItem.clean_indices!
1313
end
1414

1515
def test_csv_download
1616
FactoryBot.create_list(:log_item, 5, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_ip_country => "US")
1717
FactoryBot.create_list(:log_item, 5, :request_at => 1421413588000, :request_ip_country => "CI")
18-
LogItem.gateway.refresh_index!
18+
LogItem.refresh_indices!
1919
default_query = JSON.generate({
2020
"condition" => "AND",
2121
"rules" => [{

‎test/admin_ui/test_stats_users.rb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ class Test::AdminUi::TestStatsUsers < Minitest::Capybara::Test
99
def setup
1010
super
1111
setup_server
12-
ElasticsearchHelper.clean_es_indices(["2014-11", "2015-01", "2015-03"])
12+
LogItem.clean_indices!
1313
end
1414

1515
def test_xss_escaping_in_table
@@ -21,7 +21,7 @@ def test_xss_escaping_in_table
2121
:user_email => user.email,
2222
:user_registration_source => user.registration_source,
2323
})
24-
LogItem.gateway.refresh_index!
24+
LogItem.refresh_indices!
2525

2626
admin_login
2727
visit "/admin/#/stats/users?search=&start_at=2015-01-12&end_at=2015-01-18"
@@ -44,7 +44,7 @@ def test_csv_download
4444
:user_registration_source => user.registration_source,
4545
})
4646
FactoryBot.create_list(:log_item, 5, :request_at => 1421413588000)
47-
LogItem.gateway.refresh_index!
47+
LogItem.refresh_indices!
4848
default_query = JSON.generate({
4949
"condition" => "AND",
5050
"rules" => [{

‎test/apis/admin/stats/test_logs.rb‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@ class Test::Apis::Admin::Stats::TestLogs < Minitest::Test
77
def setup
88
super
99
setup_server
10-
ElasticsearchHelper.clean_es_indices(["2014-11", "2015-01", "2015-03"])
10+
LogItem.clean_indices!
1111
end
1212

1313
def test_strips_api_keys_from_request_url_in_json
1414
FactoryBot.create(:log_item, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_url => "http://127.0.0.1/with_api_key/?foo=bar&api_key=my_secret_key", :request_path => "/with_api_key/", :request_url_query => "foo=bar&api_key=my_secret_key", :request_query => { "foo" => "bar", "api_key" => "my_secret_key" }, :request_user_agent => unique_test_id)
15-
LogItem.refresh_index!
15+
LogItem.refresh_indices!
1616

1717
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/logs.json", http_options.deep_merge(admin_session).deep_merge({
1818
:params => {
@@ -30,13 +30,15 @@ def test_strips_api_keys_from_request_url_in_json
3030
assert_equal(1, data["recordsTotal"], data)
3131
assert_equal("/with_api_key/?foo=bar", data["data"][0]["request_url"])
3232
assert_equal("foo=bar", data["data"][0]["request_url_query"])
33-
assert_equal({ "foo" => "bar" }, data["data"][0]["request_query"])
33+
if($config["log_template_version"] < 2)
34+
assert_equal({ "foo" => "bar" }, data["data"][0]["request_query"])
35+
end
3436
refute_match("my_secret_key", body)
3537
end
3638

3739
def test_strips_api_keys_from_request_url_in_csv
3840
FactoryBot.create(:log_item, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_url => "http://127.0.0.1/with_api_key/?api_key=my_secret_key&foo=bar", :request_path => "/with_api_key/", :request_url_query => "api_key=my_secret_key&foo=bar", :request_query => { "foo" => "bar", "api_key" => "my_secret_key" }, :request_user_agent => unique_test_id)
39-
LogItem.refresh_index!
41+
LogItem.refresh_indices!
4042

4143
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/logs.csv", http_options.deep_merge(admin_session).deep_merge({
4244
:params => {
@@ -56,7 +58,7 @@ def test_strips_api_keys_from_request_url_in_csv
5658

5759
def test_downloading_csv_that_uses_scan_and_scroll_elasticsearch_query
5860
FactoryBot.create_list(:log_item, 1005, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_user_agent => unique_test_id)
59-
LogItem.refresh_index!
61+
LogItem.refresh_indices!
6062

6163
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/logs.csv", http_options.deep_merge(admin_session).deep_merge({
6264
:params => {
@@ -78,7 +80,7 @@ def test_downloading_csv_that_uses_scan_and_scroll_elasticsearch_query
7880

7981
def test_query_builder_case_insensitive_defaults
8082
FactoryBot.create(:log_item, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_user_agent => "MOZILLAAA-#{unique_test_id}")
81-
LogItem.refresh_index!
83+
LogItem.refresh_indices!
8284

8385
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/logs.json", http_options.deep_merge(admin_session).deep_merge({
8486
:params => {
@@ -99,7 +101,6 @@ def test_query_builder_case_insensitive_defaults
99101

100102
def test_query_builder_api_key_case_sensitive
101103
FactoryBot.create(:log_item, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :api_key => "AbCDeF", :request_user_agent => unique_test_id)
102-
LogItem.refresh_index!
103104

104105
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/logs.json", http_options.deep_merge(admin_session).deep_merge({
105106
:params => {
@@ -121,7 +122,7 @@ def test_query_builder_api_key_case_sensitive
121122
def test_query_builder_nulls
122123
FactoryBot.create(:log_item, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_user_agent => "#{unique_test_id}-null")
123124
FactoryBot.create(:log_item, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :gatekeeper_denied_code => "api_key_missing", :request_user_agent => "#{unique_test_id}-not-null")
124-
LogItem.refresh_index!
125+
LogItem.refresh_indices!
125126

126127
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/logs.json", http_options.deep_merge(admin_session).deep_merge({
127128
:params => {

‎test/apis/admin/stats/test_map.rb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@ class Test::Apis::Admin::Stats::TestMap < Minitest::Test
77
def setup
88
super
99
setup_server
10-
ElasticsearchHelper.clean_es_indices(["2014-11", "2015-01", "2015-03"])
10+
LogItem.clean_indices!
1111
end
1212

1313
def test_world
1414
FactoryBot.create_list(:log_item, 2, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_ip_country => "US", :request_ip_region => "CO", :request_ip_city => "Golden")
1515
FactoryBot.create_list(:log_item, 1, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_ip_country => "CA", :request_ip_region => "ON", :request_ip_city => "Toronto")
16-
LogItem.refresh_index!
16+
LogItem.refresh_indices!
1717

1818
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/map.json", http_options.deep_merge(admin_session).deep_merge({
1919
:params => {
@@ -45,7 +45,7 @@ def test_country_non_us
4545
FactoryBot.create(:log_city_location, :country => "CA", :region => "QC", :city => "Montréal", :location => { :type => "Point", :coordinates => [-73.5877, 45.5009] })
4646
FactoryBot.create_list(:log_item, 2, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_ip_country => "CA", :request_ip_region => "ON", :request_ip_city => "Toronto")
4747
FactoryBot.create_list(:log_item, 1, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_ip_country => "CA", :request_ip_region => "QC", :request_ip_city => "Montréal")
48-
LogItem.refresh_index!
48+
LogItem.refresh_indices!
4949

5050
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/map.json", http_options.deep_merge(admin_session).deep_merge({
5151
:params => {
@@ -78,7 +78,7 @@ def test_country_non_us
7878
def test_country_us
7979
FactoryBot.create_list(:log_item, 2, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_ip_country => "US", :request_ip_region => "CO", :request_ip_city => "Golden")
8080
FactoryBot.create_list(:log_item, 1, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_ip_country => "US", :request_ip_region => "CA", :request_ip_city => "San Diego")
81-
LogItem.refresh_index!
81+
LogItem.refresh_indices!
8282

8383
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/map.json", http_options.deep_merge(admin_session).deep_merge({
8484
:params => {
@@ -112,7 +112,7 @@ def test_us_state
112112
FactoryBot.create(:log_city_location, :country => "US", :region => "CO", :city => "Golden", :location => { :type => "Point", :coordinates => [-105.2433, 39.7146] })
113113
FactoryBot.create_list(:log_item, 2, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_ip_country => "US", :request_ip_region => "CO", :request_ip_city => "Golden")
114114
FactoryBot.create_list(:log_item, 1, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_ip_country => "US", :request_ip_region => "CA", :request_ip_city => "San Diego")
115-
LogItem.refresh_index!
115+
LogItem.refresh_indices!
116116

117117
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/map.json", http_options.deep_merge(admin_session).deep_merge({
118118
:params => {
@@ -144,7 +144,7 @@ def test_us_state
144144
def test_csv_download
145145
FactoryGirl.create_list(:log_item, 2, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_ip_country => "US", :request_ip_region => "CO", :request_ip_city => "Golden")
146146
FactoryGirl.create_list(:log_item, 1, :request_at => Time.parse("2015-01-16T06:06:28.816Z").utc, :request_ip_country => "CA", :request_ip_region => "ON", :request_ip_city => "Toronto")
147-
LogItem.refresh_index!
147+
LogItem.refresh_indices!
148148

149149
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/map.csv", http_options.deep_merge(admin_session).deep_merge({
150150
:params => {

‎test/apis/admin/stats/test_search.rb‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ class Test::Apis::Admin::Stats::TestSearch < Minitest::Test
77
def setup
88
super
99
setup_server
10-
ElasticsearchHelper.clean_es_indices(["2014-11", "2015-01", "2015-03"])
10+
LogItem.clean_indices!
1111
end
1212

1313
def test_bins_results_by_day_with_time_zone_support
@@ -17,7 +17,7 @@ def test_bins_results_by_day_with_time_zone_support
1717
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2015-01-18T23:59:59"))
1818
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2015-01-19T00:00:00"))
1919
end
20-
LogItem.refresh_index!
20+
LogItem.refresh_indices!
2121

2222
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/search.json", http_options.deep_merge(admin_session).deep_merge({
2323
:params => {
@@ -42,15 +42,13 @@ def test_bins_results_by_day_with_time_zone_support
4242
end
4343

4444
def test_bins_daily_results_daylight_saving_time_begin
45-
LogItem.index_name = "api-umbrella-logs-write-2015-03"
4645
Time.use_zone("UTC") do
4746
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2015-03-08T00:00:00"))
4847
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2015-03-08T08:59:59"))
4948
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2015-03-08T09:00:00"))
5049
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2015-03-09T10:00:00"))
5150
end
52-
LogItem.refresh_index!
53-
LogItem.index_name = "api-umbrella-logs-write-2015-01"
51+
LogItem.refresh_indices!
5452

5553
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/search.json", http_options.deep_merge(admin_session).deep_merge({
5654
:params => {
@@ -79,13 +77,11 @@ def test_bins_daily_results_daylight_saving_time_begin
7977
end
8078

8179
def test_bins_hourly_results_daylight_saving_time_begin
82-
LogItem.index_name = "api-umbrella-logs-write-2015-03"
8380
Time.use_zone("UTC") do
8481
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2015-03-08T08:59:59"))
8582
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2015-03-08T09:00:00"))
8683
end
87-
LogItem.refresh_index!
88-
LogItem.index_name = "api-umbrella-logs-write-2015-01"
84+
LogItem.refresh_indices!
8985

9086
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/search.json", http_options.deep_merge(admin_session).deep_merge({
9187
:params => {
@@ -118,15 +114,13 @@ def test_bins_hourly_results_daylight_saving_time_begin
118114
end
119115

120116
def test_bins_daily_results_daylight_saving_time_end
121-
LogItem.index_name = "api-umbrella-logs-write-2014-11"
122117
Time.use_zone("UTC") do
123118
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2014-11-02T00:00:00"))
124119
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2014-11-02T08:59:59"))
125120
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2014-11-02T09:00:00"))
126121
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2014-11-03T10:00:00"))
127122
end
128-
LogItem.refresh_index!
129-
LogItem.index_name = "api-umbrella-logs-write-2015-01"
123+
LogItem.refresh_indices!
130124

131125
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/search.json", http_options.deep_merge(admin_session).deep_merge({
132126
:params => {
@@ -155,13 +149,11 @@ def test_bins_daily_results_daylight_saving_time_end
155149
end
156150

157151
def test_bins_hourly_results_daylight_saving_time_end
158-
LogItem.index_name = "api-umbrella-logs-write-2014-11"
159152
Time.use_zone("UTC") do
160153
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2014-11-02T08:59:59"))
161154
FactoryBot.create(:log_item, :request_at => Time.zone.parse("2014-11-02T09:00:00"))
162155
end
163-
LogItem.refresh_index!
164-
LogItem.index_name = "api-umbrella-logs-write-2015-01"
156+
LogItem.refresh_indices!
165157

166158
response = Typhoeus.get("https://127.0.0.1:9081/admin/stats/search.json", http_options.deep_merge(admin_session).deep_merge({
167159
:params => {

0 commit comments

Comments
 (0)