Skip to content

Commit 0d23856

Browse files
simonzhaomssimonzhaoms
committed
Remove warning messages
1 parent 524cb9d commit 0d23856

4 files changed

Lines changed: 134 additions & 406 deletions

File tree

‎examples/00_quick_start/als_movielens.ipynb‎

Lines changed: 57 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@
5151
"from pyspark.sql import SparkSession\n",
5252
"from pyspark.sql.types import StructType, StructField\n",
5353
"from pyspark.sql.types import StringType, FloatType, IntegerType, LongType\n",
54+
"import warnings\n",
55+
"warnings.simplefilter(action='ignore', category=FutureWarning)\n",
5456
"\n",
5557
"from recommenders.utils.timer import Timer\n",
5658
"from recommenders.datasets import movielens\n",
@@ -106,29 +108,12 @@
106108
"cell_type": "code",
107109
"execution_count": 3,
108110
"metadata": {},
109-
"outputs": [
110-
{
111-
"name": "stderr",
112-
"output_type": "stream",
113-
"text": [
114-
"22/01/13 16:26:50 WARN Utils: Your hostname, pdfocal resolves to a loopback address: 127.0.1.1; using 10.211.55.61 instead (on interface enp0s5)\n",
115-
"22/01/13 16:26:50 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n",
116-
"WARNING: An illegal reflective access operation has occurred\n",
117-
"WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/home/test/miniconda3/envs/reco/lib/python3.8/site-packages/pyspark/jars/spark-unsafe_2.12-3.2.0.jar) to constructor java.nio.DirectByteBuffer(long,int)\n",
118-
"WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n",
119-
"WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n",
120-
"WARNING: All illegal access operations will be denied in a future release\n",
121-
"Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
122-
"Setting default log level to \"WARN\".\n",
123-
"To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
124-
"22/01/13 16:26:51 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
125-
]
126-
}
127-
],
111+
"outputs": [],
128112
"source": [
129113
"# the following settings work well for debugging locally on VM - change when running on a cluster\n",
130114
"# set up a giant single executor with many threads and specify memory cap\n",
131-
"spark = start_or_get_spark(\"ALS PySpark\", memory=\"16g\", config={\"spark.sql.analyzer.failAmbiguousSelfJoin\": \"false\"})"
115+
"spark = start_or_get_spark(\"ALS PySpark\", memory=\"16g\")\n",
116+
"spark.conf.set(\"spark.sql.analyzer.failAmbiguousSelfJoin\", \"false\")"
132117
]
133118
},
134119
{
@@ -147,7 +132,7 @@
147132
"name": "stderr",
148133
"output_type": "stream",
149134
"text": [
150-
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.81k/4.81k [00:20<00:00, 239KB/s]\n",
135+
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.81k/4.81k [00:05<00:00, 882KB/s]\n",
151136
" \r"
152137
]
153138
},
@@ -266,20 +251,11 @@
266251
"execution_count": 7,
267252
"metadata": {},
268253
"outputs": [
269-
{
270-
"name": "stderr",
271-
"output_type": "stream",
272-
"text": [
273-
"22/01/13 16:27:59 WARN InstanceBuilder$NativeBLAS: Failed to load implementation from:dev.ludovic.netlib.blas.JNIBLAS\n",
274-
"22/01/13 16:27:59 WARN InstanceBuilder$NativeBLAS: Failed to load implementation from:dev.ludovic.netlib.blas.ForeignLinkerBLAS\n",
275-
"22/01/13 16:27:59 WARN InstanceBuilder$NativeLAPACK: Failed to load implementation from:dev.ludovic.netlib.lapack.JNILAPACK\n"
276-
]
277-
},
278254
{
279255
"name": "stdout",
280256
"output_type": "stream",
281257
"text": [
282-
"Took 10.751604393999514 seconds for training.\n"
258+
"Took 7.5410127229988575 seconds for training.\n"
283259
]
284260
}
285261
],
@@ -308,15 +284,14 @@
308284
"name": "stderr",
309285
"output_type": "stream",
310286
"text": [
311-
"22/01/13 16:28:12 WARN Column: Constructing trivially true equals predicate, 'UserId#0 = UserId#0'. Perhaps you need to use aliases.\n",
312-
"[Stage 126:===================================================> (194 + 2) / 200]\r"
287+
"[Stage 126:====================================================>(198 + 2) / 200]\r"
313288
]
314289
},
315290
{
316291
"name": "stdout",
317292
"output_type": "stream",
318293
"text": [
319-
"Took 25.308568059001118 seconds for prediction.\n"
294+
"Took 25.246142672998758 seconds for prediction.\n"
320295
]
321296
},
322297
{
@@ -367,25 +342,25 @@
367342
"|UserId|MovieId|prediction|\n",
368343
"+------+-------+----------+\n",
369344
"| 1| 587| 4.1602826|\n",
370-
"| 1| 869| 2.7732866|\n",
371-
"| 1| 1208| 2.0333834|\n",
372-
"| 1| 1348| 1.0019258|\n",
373-
"| 1| 1357| 0.9430025|\n",
345+
"| 1| 869| 2.7732863|\n",
346+
"| 1| 1208| 2.033383|\n",
347+
"| 1| 1348| 1.0019257|\n",
348+
"| 1| 1357| 0.9430026|\n",
374349
"| 1| 1677| 2.8777318|\n",
375350
"| 2| 80| 2.351385|\n",
376-
"| 2| 472| 2.5865324|\n",
377-
"| 2| 582| 3.9548614|\n",
378-
"| 2| 838| 0.9482964|\n",
379-
"| 2| 975| 3.1133537|\n",
380-
"| 2| 1260| 1.9871742|\n",
351+
"| 2| 472| 2.5865319|\n",
352+
"| 2| 582| 3.9548612|\n",
353+
"| 2| 838| 0.9482963|\n",
354+
"| 2| 975| 3.1133535|\n",
355+
"| 2| 1260| 1.9871743|\n",
381356
"| 2| 1325| 1.2368056|\n",
382357
"| 2| 1381| 3.5477588|\n",
383-
"| 2| 1530| 2.0882902|\n",
384-
"| 3| 22| 3.1524534|\n",
385-
"| 3| 57| 3.6980166|\n",
386-
"| 3| 89| 3.9733818|\n",
387-
"| 3| 367| 3.6629043|\n",
388-
"| 3| 1091| 0.9144471|\n",
358+
"| 2| 1530| 2.08829|\n",
359+
"| 3| 22| 3.1524537|\n",
360+
"| 3| 57| 3.6980162|\n",
361+
"| 3| 89| 3.9733813|\n",
362+
"| 3| 367| 3.6629045|\n",
363+
"| 3| 1091| 0.9144474|\n",
389364
"+------+-------+----------+\n",
390365
"only showing top 20 rows\n",
391366
"\n"
@@ -412,8 +387,6 @@
412387
"name": "stderr",
413388
"output_type": "stream",
414389
"text": [
415-
"/home/test/miniconda3/envs/reco/lib/python3.8/site-packages/pyspark/sql/context.py:125: FutureWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.\n",
416-
" warnings.warn(\n",
417390
" \r"
418391
]
419392
}
@@ -482,10 +455,7 @@
482455
"name": "stderr",
483456
"output_type": "stream",
484457
"text": [
485-
"\r",
486-
"[Stage 499:==========> (169 + 2) / 200][Stage 500:====> (3 + 0) / 10]\r",
487-
"\r",
488-
"[Stage 499:============>(185 + 2) / 200][Stage 500:====> (3 + 0) / 10]\r"
458+
"[Stage 500:=============================================> (171 + 3) / 200]\r"
489459
]
490460
},
491461
{
@@ -495,25 +465,25 @@
495465
"+------+-------+------+---------+----------+\n",
496466
"|UserId|MovieId|Rating|Timestamp|prediction|\n",
497467
"+------+-------+------+---------+----------+\n",
498-
"| 580| 148| 4.0|884125773| 3.4059544|\n",
499-
"| 406| 148| 3.0|879540276| 2.7134616|\n",
500-
"| 916| 148| 2.0|880843892| 2.224198|\n",
501-
"| 663| 148| 4.0|889492989| 2.7143617|\n",
468+
"| 580| 148| 4.0|884125773| 3.4059548|\n",
469+
"| 406| 148| 3.0|879540276| 2.7134619|\n",
470+
"| 916| 148| 2.0|880843892| 2.2241986|\n",
471+
"| 663| 148| 4.0|889492989| 2.714362|\n",
502472
"| 330| 148| 4.0|876544781| 4.52321|\n",
503473
"| 935| 148| 4.0|884472892| 4.3838587|\n",
504-
"| 308| 148| 3.0|887740788| 2.616949|\n",
505-
"| 20| 148| 5.0|879668713| 4.37212|\n",
474+
"| 308| 148| 3.0|887740788| 2.6169493|\n",
475+
"| 20| 148| 5.0|879668713| 4.3721194|\n",
506476
"| 923| 148| 4.0|880387474| 3.9818575|\n",
507-
"| 455| 148| 3.0|879110346| 3.076419|\n",
477+
"| 455| 148| 3.0|879110346| 3.0764186|\n",
508478
"| 15| 148| 3.0|879456049| 2.9913845|\n",
509-
"| 374| 148| 4.0|880392992| 3.2223387|\n",
510-
"| 880| 148| 2.0|880167030| 2.8111987|\n",
511-
"| 677| 148| 4.0|889399265| 3.8451848|\n",
512-
"| 49| 148| 1.0|888068195| 1.3751595|\n",
479+
"| 374| 148| 4.0|880392992| 3.2223384|\n",
480+
"| 880| 148| 2.0|880167030| 2.8111982|\n",
481+
"| 677| 148| 4.0|889399265| 3.8451843|\n",
482+
"| 49| 148| 1.0|888068195| 1.3751594|\n",
513483
"| 244| 148| 2.0|880605071| 2.6781514|\n",
514484
"| 84| 148| 4.0|883452274| 3.6721768|\n",
515-
"| 627| 148| 3.0|879530463| 2.636207|\n",
516-
"| 434| 148| 3.0|886724797| 3.0973825|\n",
485+
"| 627| 148| 3.0|879530463| 2.6362069|\n",
486+
"| 434| 148| 3.0|886724797| 3.0973828|\n",
517487
"| 793| 148| 4.0|875104498| 2.2886577|\n",
518488
"+------+-------+------+---------+----------+\n",
519489
"only showing top 20 rows\n",
@@ -525,7 +495,7 @@
525495
"output_type": "stream",
526496
"text": [
527497
"\r",
528-
"[Stage 500:============================> (5 + 2) / 10]\r",
498+
"[Stage 500:=================================================> (186 + 3) / 200]\r",
529499
"\r",
530500
" \r"
531501
]
@@ -546,7 +516,7 @@
546516
"name": "stderr",
547517
"output_type": "stream",
548518
"text": [
549-
" \r"
519+
"[Stage 775:==============================================> (174 + 2) / 200]\r"
550520
]
551521
},
552522
{
@@ -559,6 +529,14 @@
559529
"Explained variance:\t0.265916\n",
560530
"R squared:\t0.259532\n"
561531
]
532+
},
533+
{
534+
"name": "stderr",
535+
"output_type": "stream",
536+
"text": [
537+
"\r",
538+
" \r"
539+
]
562540
}
563541
],
564542
"source": [
@@ -577,14 +555,6 @@
577555
"execution_count": 14,
578556
"metadata": {},
579557
"outputs": [
580-
{
581-
"name": "stderr",
582-
"output_type": "stream",
583-
"text": [
584-
"/home/test/miniconda3/envs/reco/lib/python3.8/site-packages/papermill/iorw.py:50: FutureWarning: pyarrow.HadoopFileSystem is deprecated as of 2.0.0, please use pyarrow.fs.HadoopFileSystem instead.\n",
585-
" from pyarrow import HadoopFileSystem\n"
586-
]
587-
},
588558
{
589559
"data": {
590560
"application/scrapbook.scrap.json+json": {
@@ -657,6 +627,9 @@
657627
"name": "stderr",
658628
"output_type": "stream",
659629
"text": [
630+
"\r",
631+
"[Stage 904:> (0 + 2) / 2]\r",
632+
"\r",
660633
" \r"
661634
]
662635
},
@@ -681,7 +654,7 @@
681654
{
682655
"data": {
683656
"application/scrapbook.scrap.json+json": {
684-
"data": 0.9674342218475348,
657+
"data": 0.9674342234414528,
685658
"encoder": "json",
686659
"name": "rmse",
687660
"version": 1
@@ -699,7 +672,7 @@
699672
{
700673
"data": {
701674
"application/scrapbook.scrap.json+json": {
702-
"data": 0.7533395123214025,
675+
"data": 0.7533395161385739,
703676
"encoder": "json",
704677
"name": "mae",
705678
"version": 1
@@ -724,7 +697,7 @@
724697
{
725698
"data": {
726699
"application/scrapbook.scrap.json+json": {
727-
"data": 0.2659161988067019,
700+
"data": 0.2659161968930053,
728701
"encoder": "json",
729702
"name": "exp_var",
730703
"version": 1
@@ -742,7 +715,7 @@
742715
{
743716
"data": {
744717
"application/scrapbook.scrap.json+json": {
745-
"data": 0.25953227528757383,
718+
"data": 0.2595322728476255,
746719
"encoder": "json",
747720
"name": "rsquared",
748721
"version": 1
@@ -760,7 +733,7 @@
760733
{
761734
"data": {
762735
"application/scrapbook.scrap.json+json": {
763-
"data": 10.751604393999514,
736+
"data": 7.5410127229988575,
764737
"encoder": "json",
765738
"name": "train_time",
766739
"version": 1
@@ -778,7 +751,7 @@
778751
{
779752
"data": {
780753
"application/scrapbook.scrap.json+json": {
781-
"data": 25.308568059001118,
754+
"data": 25.246142672998758,
782755
"encoder": "json",
783756
"name": "test_time",
784757
"version": 1

0 commit comments

Comments
 (0)