Spark NLP: aprenda a otimizar a linguagem natural com eficácia

pip install -q pyspark==3.3.0 spark-nlp==4.0.1

import sparknlp
spark = sparknlp.start()

data = [("The Beatles", "There are places I'll remember "+
                        "All my life though some have changed "+
                        "Some forever, not for better "+
                        "Some have gone and some remain"),
        ("Oasis", "So I start a revolution from my bed " + 
                  "Cause you said the brains I had went to my head " +
                  "Step outside, summertime's in bloom "+
                  "Stand up beside the fireplace"),
        ("Pink Floyd", "How I wish you were here " +
                        "We're just two lost soul " +
                        "Swimming in a fish bowl year after year " + 
                        "Running over the same old ground")]
df_musica = spark.createDataFrame(data, ["artista", "letra"])

from sparknlp.annotator import LemmatizerModel, Stemmer, Tokenizer, StopWordsCleaner
from sparknlp.base import DocumentAssembler

document_assembler = DocumentAssembler()\\
                       .setInputCol("letra")\\
                       .setOutputCol("document")
doc_df = document_assembler.transform(df_musica)
doc_df.select("document.result").show(truncate=False)

+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|result                                                                                                                                                 |
+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|[There are places I'll remember All my life though some have changed Some forever, not for better Some have gone and some remain]                      |
|[So I start a revolution from my bed Cause you said the brains I had went to my head Step outside, summertime's in bloom Stand up beside the fireplace]|
|[How I wish you were here We're just two lost soul Swimming in a fish bowl year after year Running over the same old ground]                           |
+-------------------------------------------------------------------------------------------------------------------------------------------------------+

tokenizer = Tokenizer()\\
              .setInputCols(["document"])\\
              .setOutputCol("token")
token_df = tokenizer.fit(doc_df).transform(doc_df)
token_df.select('token.result').show(truncate=False)

+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|result                                                                                                                                                                               |
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|[There, are, places, I'll, remember, All, my, life, though, some, have, changed, Some, forever, ,, not, for, better, Some, have, gone, and, some, remain]                            |
|[So, I, start, a, revolution, from, my, bed, Cause, you, said, the, brains, I, had, went, to, my, head, Step, outside, ,, summertime's, in, bloom, Stand, up, beside, the, fireplace]|
|[How, I, wish, you, were, here, We're, just, two, lost, soul, Swimming, in, a, fish, bowl, year, after, year, Running, over, the, same, old, ground]                                 |
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+

stopwords_cleaner = StopWordsCleaner()\\
      .setInputCols("token")\
      .setOutputCol("cleanTokens")
token_df_clean = stopwords_cleaner.transform(token_df)
token_df_clean.select("cleanTokens.result").show(truncate=False)

+--------------------------------------------------------------------------------------------------------------------------+
|result                                                                                                                    |
+--------------------------------------------------------------------------------------------------------------------------+
|[places, remember, life, though, changed, forever, ,, better, gone, remain]                                               |
|[start, revolution, bed, Cause, said, brains, went, head, Step, outside, ,, summertime's, bloom, Stand, beside, fireplace]|
|[wish, two, lost, soul, Swimming, fish, bowl, year, year, Running, old, ground]                                           |
+--------------------------------------------------------------------------------------------------------------------------+

stopwords_cleaner.getStopWords()

stemmer = Stemmer() \\
    .setInputCols(["cleanTokens"]) \\
    .setOutputCol("stem")
stem = stemmer.transform(token_df_clean)
stem.select("stem.result").show(truncate=False)

+-----------------------------------------------------------------------------------------------------------------+
|result                                                                                                           |
+-----------------------------------------------------------------------------------------------------------------+
|[place, rememb, life, though, chang, forev, ,, better, gone, remain]                                             |
|[start, revolut, bed, caus, said, brain, went, head, step, outsid, ,, summertime', bloom, stand, besid, fireplac]|
|[wish, two, lost, soul, swim, fish, bowl, year, year, run, old, ground]                                          |
+-----------------------------------------------------------------------------------------------------------------+

lemmatizer = LemmatizerModel.pretrained() \\
              .setInputCols(["cleanTokens"]) \\
              .setOutputCol("lemma")
result = lemmatizer.transform(stem)
result.select("lemma.result").show(truncate=False)

+----------------------------------------------------------------------------------------------------------------------+
|result                                                                                                                |
+----------------------------------------------------------------------------------------------------------------------+
|[place, remember, life, though, change, forever, ,, well, go, remain]                                                 |
|[start, revolution, bed, Cause, say, brain, go, head, Step, outside, ,, summertime's, bloom, Stand, beside, fireplace]|
|[wish, two, lose, soul, Swimming, fish, bowl, year, year, Running, old, ground]                                       |
+----------------------------------------------------------------------------------------------------------------------+