[LinkedIn] Add extractor (#1597)

Closes #1206 
Authored by: u-spec-png
This commit is contained in:
u-spec-png 2021-11-18 21:57:40 +00:00 committed by GitHub
parent 22a510ff44
commit 402cd603a4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 78 additions and 28 deletions

View file

@ -704,6 +704,7 @@
LineLiveChannelIE, LineLiveChannelIE,
) )
from .linkedin import ( from .linkedin import (
LinkedInIE,
LinkedInLearningIE, LinkedInLearningIE,
LinkedInLearningCourseIE, LinkedInLearningCourseIE,
) )

View file

@ -6,21 +6,56 @@
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
clean_html,
extract_attributes,
ExtractorError, ExtractorError,
float_or_none, float_or_none,
get_element_by_class,
int_or_none, int_or_none,
srt_subtitles_timecode, srt_subtitles_timecode,
strip_or_none,
mimetype2ext,
try_get, try_get,
urlencode_postdata, urlencode_postdata,
urljoin, urljoin,
) )
class LinkedInLearningBaseIE(InfoExtractor): class LinkedInBaseIE(InfoExtractor):
_NETRC_MACHINE = 'linkedin' _NETRC_MACHINE = 'linkedin'
_LOGIN_URL = 'https://www.linkedin.com/uas/login?trk=learning'
_logged_in = False _logged_in = False
def _real_initialize(self):
if self._logged_in:
return
email, password = self._get_login_info()
if email is None:
return
login_page = self._download_webpage(
self._LOGIN_URL, None, 'Downloading login page')
action_url = urljoin(self._LOGIN_URL, self._search_regex(
r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page, 'post url',
default='https://www.linkedin.com/uas/login-submit', group='url'))
data = self._hidden_inputs(login_page)
data.update({
'session_key': email,
'session_password': password,
})
login_submit_page = self._download_webpage(
action_url, None, 'Logging in',
data=urlencode_postdata(data))
error = self._search_regex(
r'<span[^>]+class="error"[^>]*>\s*(.+?)\s*</span>',
login_submit_page, 'error', default=None)
if error:
raise ExtractorError(error, expected=True)
LinkedInBaseIE._logged_in = True
class LinkedInLearningBaseIE(LinkedInBaseIE):
_LOGIN_URL = 'https://www.linkedin.com/uas/login?trk=learning'
def _call_api(self, course_slug, fields, video_slug=None, resolution=None): def _call_api(self, course_slug, fields, video_slug=None, resolution=None):
query = { query = {
'courseSlug': course_slug, 'courseSlug': course_slug,
@ -52,32 +87,47 @@ def _get_urn_id(self, video_data):
def _get_video_id(self, video_data, course_slug, video_slug): def _get_video_id(self, video_data, course_slug, video_slug):
return self._get_urn_id(video_data) or '%s/%s' % (course_slug, video_slug) return self._get_urn_id(video_data) or '%s/%s' % (course_slug, video_slug)
def _real_initialize(self):
if self._logged_in:
return
email, password = self._get_login_info()
if email is None:
return
login_page = self._download_webpage( class LinkedInIE(LinkedInBaseIE):
self._LOGIN_URL, None, 'Downloading login page') _VALID_URL = r'https?://(?:www\.)?linkedin\.com/posts/.+?(?P<id>\d+)'
action_url = urljoin(self._LOGIN_URL, self._search_regex( _TESTS = [{
r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page, 'post url', 'url': 'https://www.linkedin.com/posts/mishalkhawaja_sendinblueviews-toronto-digitalmarketing-ugcPost-6850898786781339649-mM20',
default='https://www.linkedin.com/uas/login-submit', group='url')) 'info_dict': {
data = self._hidden_inputs(login_page) 'id': '6850898786781339649',
data.update({ 'ext': 'mp4',
'session_key': email, 'title': 'Mishal K. on LinkedIn: #sendinblueviews #toronto #digitalmarketing',
'session_password': password, 'description': 'md5:be125430bab1c574f16aeb186a4d5b19',
}) 'creator': 'Mishal K.'
login_submit_page = self._download_webpage( },
action_url, None, 'Logging in', }]
data=urlencode_postdata(data))
error = self._search_regex( def _real_extract(self, url):
r'<span[^>]+class="error"[^>]*>\s*(.+?)\s*</span>', video_id = self._match_id(url)
login_submit_page, 'error', default=None) webpage = self._download_webpage(url, video_id)
if error:
raise ExtractorError(error, expected=True) title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
LinkedInLearningBaseIE._logged_in = True description = clean_html(get_element_by_class('share-update-card__update-text', webpage))
like_count = int_or_none(get_element_by_class('social-counts-reactions__social-counts-numRections', webpage))
creator = strip_or_none(clean_html(get_element_by_class('comment__actor-name', webpage)))
sources = self._parse_json(extract_attributes(self._search_regex(r'(<video[^>]+>)', webpage, 'video'))['data-sources'], video_id)
formats = [{
'url': source['src'],
'ext': mimetype2ext(source.get('type')),
'tbr': float_or_none(source.get('data-bitrate'), scale=1000),
} for source in sources]
self._sort_formats(formats)
return {
'id': video_id,
'formats': formats,
'title': title,
'like_count': like_count,
'creator': creator,
'thumbnail': self._og_search_thumbnail(webpage),
'description': description,
}
class LinkedInLearningIE(LinkedInLearningBaseIE): class LinkedInLearningIE(LinkedInLearningBaseIE):
@ -108,7 +158,6 @@ def json2srt(self, transcript_lines, duration=None):
def _real_extract(self, url): def _real_extract(self, url):
course_slug, video_slug = self._match_valid_url(url).groups() course_slug, video_slug = self._match_valid_url(url).groups()
video_data = None
formats = [] formats = []
for width, height in ((640, 360), (960, 540), (1280, 720)): for width, height in ((640, 360), (960, 540), (1280, 720)):
video_data = self._call_api( video_data = self._call_api(